Skip to content

Commit 8520d57

Browse files
mason-sharpclaude
authored andcommitted
Fix precision loss for large bigint PKs in table repair JSON roundtrip
OrderedMap.UnmarshalJSON used Go's default JSON number decoding (float64), which silently truncates integers exceeding 2^53. For tables with large bigint primary keys (e.g. snowflake IDs like 415588913294348289), this caused: - PK corruption: 415588913294348289 → 415588913294348288 (off by 1) - PK collisions: adjacent PKs mapped to the same float64, causing rows to silently overwrite each other in repair maps - Wrong upserts/deletes: repairs targeted wrong rows or missed them - Growing diffs after repair: corrupted values replicated via spock Fix: add dec.UseNumber() to OrderedMap.UnmarshalJSON so JSON numbers are preserved as json.Number strings. Update ConvertToPgxType to handle json.Number for integer types (lossless Int64 parse), numeric/decimal (exact string via pgtype.Numeric), and float types (Float64 parse). Replace the three duplicate lossy numeric helpers (toFloat64, asFloat, asNumber) with a single precision-safe CompareNumeric in pkg/common that uses an int64 fast path and falls back to math/big.Float (256-bit) for json.Number decimals, large uint64, and native floats. Also handle json.Number in repair plan when-expression literals (scanNumber), pk_in matching, and origin timestamp extraction. The v4 pgtype dependency will be removed in a follow-on PR. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 005e1e0 commit 8520d57

8 files changed

Lines changed: 545 additions & 133 deletions

File tree

internal/consistency/repair/executor.go

Lines changed: 14 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -286,10 +286,8 @@ func pkMatchesOverride(pkOrder []string, rowPk map[string]any, overridePk map[st
286286
}
287287

288288
equal := func(a, b any) bool {
289-
if af, ok := asFloat(a); ok {
290-
if bf, ok2 := asFloat(b); ok2 {
291-
return af == bf
292-
}
289+
if cmp, ok := utils.CompareNumeric(a, b); ok {
290+
return cmp == 0
293291
}
294292
return reflect.DeepEqual(a, b)
295293
}
@@ -308,10 +306,8 @@ func matchPKIn(matchers []planner.RepairPKMatcher, rowPk map[string]any, pkOrder
308306
}
309307

310308
equal := func(a, b any) bool {
311-
if af, ok := asFloat(a); ok {
312-
if bf, ok2 := asFloat(b); ok2 {
313-
return af == bf
314-
}
309+
if cmp, ok := utils.CompareNumeric(a, b); ok {
310+
return cmp == 0
315311
}
316312
return reflect.DeepEqual(a, b)
317313
}
@@ -353,11 +349,10 @@ func matchPKIn(matchers []planner.RepairPKMatcher, rowPk map[string]any, pkOrder
353349
}
354350

355351
func compareRange(val, from, to any) bool {
356-
ln, lok := asFloat(val)
357-
fn, fok := asFloat(from)
358-
tn, tok := asFloat(to)
359-
if lok && fok && tok {
360-
return ln >= fn && ln <= tn
352+
cmpFrom, okFrom := utils.CompareNumeric(val, from)
353+
cmpTo, okTo := utils.CompareNumeric(val, to)
354+
if okFrom && okTo {
355+
return cmpFrom >= 0 && cmpTo <= 0
361356
}
362357

363358
ls, lsok := val.(string)
@@ -369,37 +364,6 @@ func compareRange(val, from, to any) bool {
369364
return false
370365
}
371366

372-
func asFloat(v any) (float64, bool) {
373-
switch n := v.(type) {
374-
case int:
375-
return float64(n), true
376-
case int8:
377-
return float64(n), true
378-
case int16:
379-
return float64(n), true
380-
case int32:
381-
return float64(n), true
382-
case int64:
383-
return float64(n), true
384-
case uint:
385-
return float64(n), true
386-
case uint8:
387-
return float64(n), true
388-
case uint16:
389-
return float64(n), true
390-
case uint32:
391-
return float64(n), true
392-
case uint64:
393-
return float64(n), true
394-
case float32:
395-
return float64(n), true
396-
case float64:
397-
return n, true
398-
default:
399-
return 0, false
400-
}
401-
}
402-
403367
func columnsIntersect(a, b []string) bool {
404368
if len(a) == 0 || len(b) == 0 {
405369
return false
@@ -780,15 +744,13 @@ func freshestSide(key string, tie string, row planDiffRow) string {
780744
return strings.TrimSpace(strings.ToLower(tie))
781745
}
782746

783-
if f1, ok := asFloat(k1); ok {
784-
if f2, ok2 := asFloat(k2); ok2 {
785-
if f1 > f2 {
786-
return "n1"
787-
} else if f2 > f1 {
788-
return "n2"
789-
}
790-
return strings.TrimSpace(strings.ToLower(tie))
747+
if cmp, ok := utils.CompareNumeric(k1, k2); ok {
748+
if cmp > 0 {
749+
return "n1"
750+
} else if cmp < 0 {
751+
return "n2"
791752
}
753+
return strings.TrimSpace(strings.ToLower(tie))
792754
}
793755

794756
if s1, ok := k1.(string); ok {

internal/consistency/repair/plan/parser/parser.go

Lines changed: 15 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,12 @@
1212
package parser
1313

1414
import (
15+
"encoding/json"
1516
"fmt"
16-
"strconv"
1717
"strings"
1818
"unicode"
19+
20+
utils "github.com/pgedge/ace/pkg/common"
1921
)
2022

2123
// WhenExpr represents a compiled predicate over diff row values (n1./n2.).
@@ -227,11 +229,13 @@ func (l *lexer) scanNumber() (token, error) {
227229
l.pos++
228230
}
229231
text := l.input[start:l.pos]
230-
num, err := strconv.ParseFloat(text, 64)
231-
if err != nil {
232+
// Store as json.Number to preserve full precision for large integers
233+
// and high-precision decimals. CompareNumeric handles json.Number natively.
234+
n := json.Number(text)
235+
if _, err := n.Float64(); err != nil {
232236
return token{}, fmt.Errorf("invalid number %q at pos %d", text, start)
233237
}
234-
return token{typ: tokNumber, lit: text, pos: start, value: num}, nil
238+
return token{typ: tokNumber, lit: text, pos: start, value: n}, nil
235239
}
236240

237241
func (l *lexer) scanIdent() (token, error) {
@@ -612,22 +616,20 @@ func compareValues(op tokenType, left any, right any) (bool, error) {
612616
}
613617
}
614618

615-
ln, lok := asNumber(left)
616-
rn, rok := asNumber(right)
617-
if lok && rok {
619+
if cmp, ok := utils.CompareNumeric(left, right); ok {
618620
switch op {
619621
case tokEq:
620-
return ln == rn, nil
622+
return cmp == 0, nil
621623
case tokNeq:
622-
return ln != rn, nil
624+
return cmp != 0, nil
623625
case tokLt:
624-
return ln < rn, nil
626+
return cmp < 0, nil
625627
case tokLte:
626-
return ln <= rn, nil
628+
return cmp <= 0, nil
627629
case tokGt:
628-
return ln > rn, nil
630+
return cmp > 0, nil
629631
case tokGte:
630-
return ln >= rn, nil
632+
return cmp >= 0, nil
631633
default:
632634
return false, fmt.Errorf("unsupported numeric operator %v", op)
633635
}
@@ -670,33 +672,3 @@ func compareValues(op tokenType, left any, right any) (bool, error) {
670672
return false, fmt.Errorf("cannot compare values of different or unsupported types (%T vs %T)", left, right)
671673
}
672674

673-
func asNumber(v any) (float64, bool) {
674-
switch n := v.(type) {
675-
case int:
676-
return float64(n), true
677-
case int8:
678-
return float64(n), true
679-
case int16:
680-
return float64(n), true
681-
case int32:
682-
return float64(n), true
683-
case int64:
684-
return float64(n), true
685-
case uint:
686-
return float64(n), true
687-
case uint8:
688-
return float64(n), true
689-
case uint16:
690-
return float64(n), true
691-
case uint32:
692-
return float64(n), true
693-
case uint64:
694-
return float64(n), true
695-
case float32:
696-
return float64(n), true
697-
case float64:
698-
return n, true
699-
default:
700-
return 0, false
701-
}
702-
}

internal/consistency/repair/table_repair.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2258,6 +2258,13 @@ func extractOriginInfoFromRow(row types.OrderedMap) *rowOriginInfo {
22582258
ts = parseNumericTimestamp(int64(v))
22592259
case float64:
22602260
ts = parseNumericTimestamp(int64(v))
2261+
case json.Number:
2262+
if i64, err := v.Int64(); err == nil {
2263+
ts = parseNumericTimestamp(i64)
2264+
} else {
2265+
logger.Warn("extractOriginInfoFromRow: failed to parse json.Number commit_ts %q: %v", v.String(), err)
2266+
return nil
2267+
}
22612268
default:
22622269
logger.Warn("extractOriginInfoFromRow: unhandled commit_ts type %T, skipping origin preservation", tsVal)
22632270
return nil

0 commit comments

Comments
 (0)