Skip to content

Commit b1b766a

Browse files
joeblack2kjoeblack2k
committed
Improve validation repair and device control
1 parent da9f842 commit b1b766a

13 files changed

Lines changed: 1045 additions & 74 deletions

‎backend/cmd/server/agent_api_handlers.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@ func mountAgentRoutes(r chi.Router, app *app) {
6868

6969
r.Get("/validation", app.handleValidationStatus)
7070
r.Post("/validation/rescan", app.handleValidationRescan)
71+
r.Post("/validation/quarantine/{id}/retry", app.handleValidationQuarantineRetry)
72+
r.Delete("/validation/quarantine/{id}", app.handleValidationQuarantineDelete)
7173

7274
r.Get("/modules", app.handleModulesList)
7375
r.Post("/modules/sync", app.handleModulesSync)

‎backend/cmd/server/save_validation.go‎

Lines changed: 102 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ type saveUploadPreviewItem struct {
2828
Format string `json:"format,omitempty"`
2929
MediaType string `json:"mediaType,omitempty"`
3030
RuntimeProfile string `json:"runtimeProfile,omitempty"`
31+
ROMSHA1 string `json:"romSha1,omitempty"`
32+
ROMMD5 string `json:"romMd5,omitempty"`
3133
SizeBytes int `json:"sizeBytes"`
3234
SHA256 string `json:"sha256"`
3335
ParserLevel string `json:"parserLevel,omitempty"`
@@ -47,28 +49,35 @@ type saveUploadPreviewResponse struct {
4749
}
4850

4951
type quarantineRecord struct {
50-
ID string `json:"id"`
51-
Filename string `json:"filename"`
52-
SourcePath string `json:"sourcePath,omitempty"`
53-
PayloadFile string `json:"payloadFile"`
54-
SizeBytes int `json:"sizeBytes"`
55-
SHA256 string `json:"sha256"`
56-
Reason string `json:"reason"`
57-
SystemSlug string `json:"systemSlug,omitempty"`
58-
DisplayTitle string `json:"displayTitle,omitempty"`
59-
ParserLevel string `json:"parserLevel,omitempty"`
60-
TrustLevel string `json:"trustLevel,omitempty"`
61-
UploadedAt time.Time `json:"uploadedAt"`
62-
UploadSource string `json:"uploadSource,omitempty"`
52+
ID string `json:"id"`
53+
Filename string `json:"filename"`
54+
SourcePath string `json:"sourcePath,omitempty"`
55+
PayloadFile string `json:"payloadFile"`
56+
SizeBytes int `json:"sizeBytes"`
57+
SHA256 string `json:"sha256"`
58+
Reason string `json:"reason"`
59+
SystemSlug string `json:"systemSlug,omitempty"`
60+
Format string `json:"format,omitempty"`
61+
MediaType string `json:"mediaType,omitempty"`
62+
RuntimeProfile string `json:"runtimeProfile,omitempty"`
63+
ROMSHA1 string `json:"romSha1,omitempty"`
64+
ROMMD5 string `json:"romMd5,omitempty"`
65+
DisplayTitle string `json:"displayTitle,omitempty"`
66+
ParserLevel string `json:"parserLevel,omitempty"`
67+
TrustLevel string `json:"trustLevel,omitempty"`
68+
UploadedAt time.Time `json:"uploadedAt"`
69+
UploadSource string `json:"uploadSource,omitempty"`
6370
}
6471

6572
type validationStatus struct {
66-
GeneratedAt time.Time `json:"generatedAt"`
67-
Counts map[string]int `json:"counts"`
68-
Systems map[string]int `json:"systems"`
69-
QuarantineCount int `json:"quarantineCount"`
70-
Quarantine []quarantineRecord `json:"quarantine"`
71-
LastRescan *saveRescanResult `json:"lastRescan,omitempty"`
73+
GeneratedAt time.Time `json:"generatedAt"`
74+
Counts map[string]int `json:"counts"`
75+
Systems map[string]int `json:"systems"`
76+
QuarantineCount int `json:"quarantineCount"`
77+
Quarantine []quarantineRecord `json:"quarantine"`
78+
CoverageSummary validationCoverageSummary `json:"coverageSummary"`
79+
Coverage []validationCoverageRecord `json:"coverage"`
80+
LastRescan *saveRescanResult `json:"lastRescan,omitempty"`
7281
}
7382

7483
func (a *app) handleSavesPreview(w http.ResponseWriter, r *http.Request) {
@@ -221,6 +230,8 @@ func previewItemFromNormalized(filename, sourcePath string, payload []byte, runt
221230
Format: strings.TrimSpace(input.Format),
222231
MediaType: strings.TrimSpace(input.MediaType),
223232
RuntimeProfile: firstNonEmpty(strings.TrimSpace(input.RuntimeProfile), runtimeProfile),
233+
ROMSHA1: strings.TrimSpace(input.ROMSHA1),
234+
ROMMD5: strings.TrimSpace(input.ROMMD5),
224235
SizeBytes: len(payload),
225236
SHA256: hex.EncodeToString(sum[:]),
226237
Reason: strings.TrimSpace(preview.RejectReason),
@@ -268,6 +279,8 @@ func rejectedPreviewItem(filename, sourcePath string, payload []byte, systemSlug
268279
SystemSlug: canonicalSegment(systemSlug, "unknown-system"),
269280
Format: strings.TrimSpace(format),
270281
RuntimeProfile: strings.TrimSpace(runtimeProfile),
282+
ROMSHA1: "",
283+
ROMMD5: "",
271284
SizeBytes: len(payload),
272285
SHA256: hex.EncodeToString(sum[:]),
273286
ParserLevel: saveParserLevelNone,
@@ -299,19 +312,24 @@ func (a *app) quarantineRejectedUpload(filename, sourcePath string, payload []by
299312
}
300313
payloadFile := "payload" + ext
301314
record := quarantineRecord{
302-
ID: filepath.Base(dir),
303-
Filename: safeName,
304-
SourcePath: strings.TrimSpace(strings.ReplaceAll(sourcePath, "\\", "/")),
305-
PayloadFile: payloadFile,
306-
SizeBytes: len(payload),
307-
SHA256: shaHex,
308-
Reason: firstNonEmpty(strings.TrimSpace(item.Reason), errUnsupportedSaveFormat.Error()),
309-
SystemSlug: strings.TrimSpace(item.SystemSlug),
310-
DisplayTitle: strings.TrimSpace(item.DisplayTitle),
311-
ParserLevel: firstNonEmpty(strings.TrimSpace(item.ParserLevel), saveParserLevelNone),
312-
TrustLevel: firstNonEmpty(strings.TrimSpace(item.TrustLevel), validationTrustLevel(item.ParserLevel)),
313-
UploadedAt: now,
314-
UploadSource: strings.TrimSpace(uploadSource),
315+
ID: filepath.Base(dir),
316+
Filename: safeName,
317+
SourcePath: strings.TrimSpace(strings.ReplaceAll(sourcePath, "\\", "/")),
318+
PayloadFile: payloadFile,
319+
SizeBytes: len(payload),
320+
SHA256: shaHex,
321+
Reason: firstNonEmpty(strings.TrimSpace(item.Reason), errUnsupportedSaveFormat.Error()),
322+
SystemSlug: strings.TrimSpace(item.SystemSlug),
323+
Format: strings.TrimSpace(item.Format),
324+
MediaType: strings.TrimSpace(item.MediaType),
325+
RuntimeProfile: strings.TrimSpace(item.RuntimeProfile),
326+
ROMSHA1: strings.TrimSpace(item.ROMSHA1),
327+
ROMMD5: strings.TrimSpace(item.ROMMD5),
328+
DisplayTitle: strings.TrimSpace(item.DisplayTitle),
329+
ParserLevel: firstNonEmpty(strings.TrimSpace(item.ParserLevel), saveParserLevelNone),
330+
TrustLevel: firstNonEmpty(strings.TrimSpace(item.TrustLevel), validationTrustLevel(item.ParserLevel)),
331+
UploadedAt: now,
332+
UploadSource: strings.TrimSpace(uploadSource),
315333
}
316334
if err := writeFileAtomic(filepath.Join(dir, payloadFile), payload, 0o644); err != nil {
317335
return
@@ -373,18 +391,25 @@ func (a *app) buildValidationStatus(lastRescan *saveRescanResult) (validationSta
373391
"unknown": 0,
374392
}
375393
systems := map[string]int{}
394+
coverage := make([]validationCoverageRecord, 0, len(records))
395+
coverageSummary := validationCoverageSummary{Total: len(records)}
376396
for _, record := range records {
377397
slug := canonicalSegment(saveRecordSystemSlug(record), "unknown-system")
378398
systems[slug]++
379399
level := saveParserLevelNone
380400
trust := ""
401+
parserID := ""
402+
gameplayFactCount := 0
381403
if record.Summary.Inspection != nil {
382404
level = strings.TrimSpace(record.Summary.Inspection.ParserLevel)
383405
trust = strings.TrimSpace(record.Summary.Inspection.TrustLevel)
406+
parserID = strings.TrimSpace(record.Summary.Inspection.ParserID)
407+
gameplayFactCount = validationGameplayFactCount(record.Summary.Inspection.SemanticFields)
384408
}
385409
switch validationBucket(level, trust) {
386410
case "semantic":
387411
counts["semanticVerified"]++
412+
coverageSummary.Semantic++
388413
case "structure":
389414
counts["structureVerified"]++
390415
case "rom":
@@ -394,7 +419,49 @@ func (a *app) buildValidationStatus(lastRescan *saveRescanResult) (validationSta
394419
default:
395420
counts["unknown"]++
396421
}
397-
}
422+
cheatsSupported := record.Summary.Cheats != nil && record.Summary.Cheats.Supported && record.Summary.Cheats.AvailableCount > 0
423+
if cheatsSupported {
424+
coverageSummary.Cheats++
425+
}
426+
if gameplayFactCount > 0 {
427+
coverageSummary.GameplayFacts++
428+
} else {
429+
coverageSummary.Missing++
430+
}
431+
systemName := ""
432+
if record.Summary.Game.System != nil {
433+
systemName = record.Summary.Game.System.Name
434+
}
435+
cheatCount := 0
436+
if record.Summary.Cheats != nil {
437+
cheatCount = record.Summary.Cheats.AvailableCount
438+
}
439+
coverage = append(coverage, validationCoverageRecord{
440+
SaveID: record.Summary.ID,
441+
DisplayTitle: firstNonEmpty(record.Summary.DisplayTitle, record.Summary.Game.DisplayTitle, record.Summary.Game.Name, record.Summary.Filename),
442+
SystemSlug: slug,
443+
SystemName: systemName,
444+
ParserLevel: level,
445+
ParserID: parserID,
446+
TrustLevel: trust,
447+
GameplayFactCount: gameplayFactCount,
448+
HasGameplayFacts: gameplayFactCount > 0,
449+
CheatsSupported: cheatsSupported,
450+
CheatCount: cheatCount,
451+
UpdatedAt: record.Summary.CreatedAt,
452+
})
453+
}
454+
sort.Slice(coverage, func(i, j int) bool {
455+
leftMissing := !coverage[i].HasGameplayFacts && !coverage[i].CheatsSupported
456+
rightMissing := !coverage[j].HasGameplayFacts && !coverage[j].CheatsSupported
457+
if leftMissing != rightMissing {
458+
return leftMissing
459+
}
460+
if coverage[i].SystemSlug != coverage[j].SystemSlug {
461+
return coverage[i].SystemSlug < coverage[j].SystemSlug
462+
}
463+
return strings.ToLower(coverage[i].DisplayTitle) < strings.ToLower(coverage[j].DisplayTitle)
464+
})
398465
quarantine, err := a.listQuarantineRecords()
399466
if err != nil {
400467
return validationStatus{}, err
@@ -405,6 +472,8 @@ func (a *app) buildValidationStatus(lastRescan *saveRescanResult) (validationSta
405472
Systems: systems,
406473
QuarantineCount: len(quarantine),
407474
Quarantine: quarantine,
475+
CoverageSummary: coverageSummary,
476+
Coverage: coverage,
408477
LastRescan: lastRescan,
409478
}, nil
410479
}

‎backend/cmd/server/save_validation_coverage.go‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"regexp"
6+
"strings"
7+
"time"
8+
)
9+
10+
type validationCoverageSummary struct {
11+
Total int `json:"total"`
12+
GameplayFacts int `json:"gameplayFacts"`
13+
Semantic int `json:"semantic"`
14+
Cheats int `json:"cheats"`
15+
Missing int `json:"missing"`
16+
}
17+
18+
type validationCoverageRecord struct {
19+
SaveID string `json:"saveId"`
20+
DisplayTitle string `json:"displayTitle"`
21+
SystemSlug string `json:"systemSlug"`
22+
SystemName string `json:"systemName,omitempty"`
23+
ParserLevel string `json:"parserLevel,omitempty"`
24+
ParserID string `json:"parserId,omitempty"`
25+
TrustLevel string `json:"trustLevel,omitempty"`
26+
GameplayFactCount int `json:"gameplayFactCount"`
27+
HasGameplayFacts bool `json:"hasGameplayFacts"`
28+
CheatsSupported bool `json:"cheatsSupported"`
29+
CheatCount int `json:"cheatCount,omitempty"`
30+
UpdatedAt time.Time `json:"updatedAt"`
31+
}
32+
33+
func validationGameplayFactCount(fields map[string]any) int {
34+
count := 0
35+
for key, value := range fields {
36+
if !validationLooksLikeGameplayFact(key, value) {
37+
continue
38+
}
39+
count++
40+
}
41+
return count
42+
}
43+
44+
func validationLooksLikeGameplayFact(key string, value any) bool {
45+
if value == nil || strings.TrimSpace(key) == "" {
46+
return false
47+
}
48+
normalized := strings.ToLower(strings.TrimSpace(key))
49+
if validationTechnicalSemanticKeys[normalized] {
50+
return false
51+
}
52+
if validationTechnicalSemanticPattern.MatchString(normalized) {
53+
return false
54+
}
55+
if _, ok := value.(map[string]any); ok {
56+
return false
57+
}
58+
if items, ok := value.([]any); ok && len(items) > 12 {
59+
return false
60+
}
61+
text := strings.TrimSpace(fmt.Sprint(value))
62+
return text != "" && len(text) <= 180
63+
}
64+
65+
var validationTechnicalSemanticKeys = map[string]bool{
66+
"blankcheck": true, "container": true, "copy1magicvalid": true, "copy1nonzerodata": true,
67+
"copy2magicvalid": true, "copy2nonzerodata": true, "databytespercopy": true,
68+
"editablefields": true, "embeddedfilename": true, "entrycount": true, "extension": true,
69+
"format": true, "layout": true, "mediatype": true, "nonffbytes": true, "nonpaddingbytes": true,
70+
"nonzerobytes": true, "rawsavekind": true, "romlinked": true, "romsha1present": true,
71+
"semanticdecoderstate": true, "sourcepath": true, "titlecode": true,
72+
"validbackupslots": true, "validcopies": true, "validprimaryslots": true,
73+
"verificationlevels": true, "wordswapped": true,
74+
}
75+
76+
var validationTechnicalSemanticPattern = regexp.MustCompile(`(?i)(checksum|crc|magic|offset|bytes|length|payload|source|sha|hash|copy|valid|verified|signature|container|extension|format|media|rom|parser|raw|nonzero|nonff)`)

0 commit comments

Comments
 (0)