Skip to content

Add support for Unicode digit normalization in string to Int and BigI… #2093

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nickeskov merged 5 commits into from
May 19, 2026
Merged

Add support for Unicode digit normalization in string to Int and BigI… #2093

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
1947619
Add support for Unicode digit normalization in string to Int and BigI…
alexeykiselev May 18, 2026
a117a09
Review issues fixed.
alexeykiselev May 18, 2026
1f5a740
Merge branch 'master' into fix-ride-unicode-string-to-numbers
alexeykiselev May 18, 2026
2139c47
Merge branch 'master' into fix-ride-unicode-string-to-numbers
alexeykiselev May 19, 2026
98392a1
Add logging of Eth transaction ID.
alexeykiselev May 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion pkg/proto/eth_transaction.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,11 @@ func (tx *EthereumTransaction) Verify() (*EthereumPublicKey, error) {
signer := MakeEthereumSigner(tx.ChainId())
senderPK, err := signer.SenderPK(tx)
if err != nil {
return nil, errors.Wrap(err, "failed to verify EthereumTransaction")
idStr := "n/a"
if tx.ID != nil {
idStr = tx.ID.String()
}
return nil, errors.Wrapf(err, "failed to verify EthereumTransaction '%s'", idStr)
}
tx.threadSafeSetSenderPK(senderPK)
return senderPK, nil
Expand Down
12 changes: 10 additions & 2 deletions pkg/ride/functions_bigint.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package ride
import (
"math/big"
"sort"
"unicode/utf8"

"github.com/ericlagergren/decimal"
"github.com/pkg/errors"
Expand All @@ -11,6 +12,8 @@ import (
"github.com/wavesplatform/gowaves/pkg/util/common"
)

const base10 = 10

var (
zeroBigInt = big.NewInt(0)
)
Expand Down Expand Up @@ -516,10 +519,15 @@ func stringToBigInt(_ environment, args ...rideType) (rideType, error) {
if err != nil {
return nil, errors.Wrap(err, "stringToBigInt")
}
if l := len(s); l > 155 { // 155 symbols is the length of math.MinBigInt value is string representation
const maxBitIntStringSize = 155 // The maximum allowed length of math.MinBigInt string.
if l := utf8.RuneCountInString(string(s)); l > maxBitIntStringSize {
return nil, errors.Errorf("stringToBigInt: string is too long (%d symbols) for a BigInt", l)
}
r, ok := new(big.Int).SetString(string(s), 10)
ns, err := normalizeDigits(string(s))
if err != nil {
return nil, errors.Wrap(err, "stringToBigInt")
}
r, ok := new(big.Int).SetString(ns, base10)
Comment thread
nickeskov marked this conversation as resolved.
if !ok {
return nil, errors.Errorf("stringToBigInt: failed to convert string '%s' to BigInt", s)
}
Expand Down
74 changes: 58 additions & 16 deletions pkg/ride/functions_bigint_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -691,7 +691,7 @@ func BenchmarkBigIntToString(b *testing.B) {
func TestStringToBigInt(t *testing.T) {
v, ok := new(big.Int).SetString("52785833603464895924505196455835395749861094195642486808108138863402869537852026544579466671752822414281401856143643660416162921950916138504990605852480", 10)
require.True(t, ok)
for _, test := range []struct {
for i, test := range []struct {
args []rideType
fail bool
r rideType
Expand All @@ -706,24 +706,45 @@ func TestStringToBigInt(t *testing.T) {
{[]rideType{rideString("9223372036854775807")}, false, toRideBigInt(math.MaxInt64)},
{[]rideType{rideString("-9223372036854775808")}, false, toRideBigInt(math.MinInt64)},
{[]rideType{rideString("52785833603464895924505196455835395749861094195642486808108138863402869537852026544579466671752822414281401856143643660416162921950916138504990605852480")}, false, rideBigInt{v: v}},
{[]rideType{rideString("⓪①②③④⑤⑥⑦⑧⑨⑩⑪⑫⑬⑭⑮⑯⑰⑱⑲⑳")}, true, nil},
{[]rideType{rideString("ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩ")}, true, nil},
{[]rideType{rideString("ⅰⅱⅲⅳⅴⅵⅶⅷⅸⅹ")}, true, nil},
{[]rideType{rideString("⁰¹²³⁴⁵⁶⁷⁸⁹")}, true, nil},
{[]rideType{rideString("₀₁₂₃₄₅₆₇₈₉")}, true, nil},
{[]rideType{rideString("𝟎𝟏𝟐𝟑𝟒𝟓𝟔𝟕𝟖𝟗")}, true, nil},
{[]rideType{rideString("𝟘𝟙𝟚𝟛𝟜𝟝𝟞𝟟𝟠𝟡")}, true, nil},
{[]rideType{rideString("𝟢𝟣𝟤𝟥𝟦𝟧𝟨𝟩𝟪𝟫")}, true, nil},
{[]rideType{rideString("𝟬𝟭𝟮𝟯𝟰𝟱𝟲𝟳𝟴𝟵")}, true, nil},
{[]rideType{rideString("𝟶𝟷𝟸𝟹𝟺𝟻𝟼𝟽𝟾𝟿")}, true, nil},
{[]rideType{rideString("０１２３４５６７８９")}, false, toRideBigInt(123456789)},
{[]rideType{rideString("٠١٢٣٤٥٦٧٨٩")}, false, toRideBigInt(123456789)},
{[]rideType{rideString("۰۱۲۳۴۵۶۷۸۹")}, false, toRideBigInt(123456789)},
{[]rideType{rideString("०१२३४५६७८९")}, false, toRideBigInt(123456789)},
{[]rideType{rideString("০১২৩৪৫৬৭৮৯")}, false, toRideBigInt(123456789)},
{[]rideType{rideString("๐๑๒๓๔๕๖๗๘๙")}, false, toRideBigInt(123456789)},
{[]rideType{rideString("௦௧௨௩௪௫௬௭௮௯")}, false, toRideBigInt(123456789)},
{[]rideType{rideString("၀၁၂၃၄၅၆၇၈၉")}, false, toRideBigInt(123456789)},
{[]rideType{rideString("០១២៣៤៥៦៧៨៩")}, false, toRideBigInt(123456789)},
{[]rideType{rideString("0"), rideInt(4)}, true, nil},
{[]rideType{rideInt(0)}, true, nil},
{[]rideType{}, true, nil},
} {
r, err := stringToBigInt(nil, test.args...)
if test.fail {
assert.Error(t, err)
} else {
require.NoError(t, err)
assert.True(t, test.r.eq(r), fmt.Sprintf("%s != %s", test.r, r))
}
t.Run(fmt.Sprintf("test_%d", i+1), func(t *testing.T) {
r, err := stringToBigInt(nil, test.args...)
if test.fail {
assert.Error(t, err)
} else {
require.NoError(t, err)
assert.True(t, test.r.eq(r), fmt.Sprintf("%s != %s", test.r, r))
}
})
}
}

func TestStringToBigIntOpt(t *testing.T) {
v, ok := new(big.Int).SetString("52785833603464895924505196455835395749861094195642486808108138863402869537852026544579466671752822414281401856143643660416162921950916138504990605852480", 10)
require.True(t, ok)
for _, test := range []struct {
for i, test := range []struct {
args []rideType
fail bool
r rideType
Expand All @@ -738,17 +759,38 @@ func TestStringToBigIntOpt(t *testing.T) {
{[]rideType{rideString("9223372036854775807")}, false, toRideBigInt(math.MaxInt64)},
{[]rideType{rideString("-9223372036854775808")}, false, toRideBigInt(math.MinInt64)},
{[]rideType{rideString("52785833603464895924505196455835395749861094195642486808108138863402869537852026544579466671752822414281401856143643660416162921950916138504990605852480")}, false, rideBigInt{v: v}},
{[]rideType{rideString("⓪①②③④⑤⑥⑦⑧⑨⑩⑪⑫⑬⑭⑮⑯⑰⑱⑲⑳")}, false, newUnit(nil)},
{[]rideType{rideString("ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩ")}, false, newUnit(nil)},
{[]rideType{rideString("ⅰⅱⅲⅳⅴⅵⅶⅷⅸⅹ")}, false, newUnit(nil)},
{[]rideType{rideString("⁰¹²³⁴⁵⁶⁷⁸⁹")}, false, newUnit(nil)},
{[]rideType{rideString("₀₁₂₃₄₅₆₇₈₉")}, false, newUnit(nil)},
{[]rideType{rideString("𝟎𝟏𝟐𝟑𝟒𝟓𝟔𝟕𝟖𝟗")}, false, newUnit(nil)},
{[]rideType{rideString("𝟘𝟙𝟚𝟛𝟜𝟝𝟞𝟟𝟠𝟡")}, false, newUnit(nil)},
{[]rideType{rideString("𝟢𝟣𝟤𝟥𝟦𝟧𝟨𝟩𝟪𝟫")}, false, newUnit(nil)},
{[]rideType{rideString("𝟬𝟭𝟮𝟯𝟰𝟱𝟲𝟳𝟴𝟵")}, false, newUnit(nil)},
{[]rideType{rideString("𝟶𝟷𝟸𝟹𝟺𝟻𝟼𝟽𝟾𝟿")}, false, newUnit(nil)},
{[]rideType{rideString("０１２３４５６７８９")}, false, toRideBigInt(123456789)},
{[]rideType{rideString("٠١٢٣٤٥٦٧٨٩")}, false, toRideBigInt(123456789)},
{[]rideType{rideString("۰۱۲۳۴۵۶۷۸۹")}, false, toRideBigInt(123456789)},
{[]rideType{rideString("०१२३४५६७८९")}, false, toRideBigInt(123456789)},
{[]rideType{rideString("০১২৩৪৫৬৭৮৯")}, false, toRideBigInt(123456789)},
{[]rideType{rideString("๐๑๒๓๔๕๖๗๘๙")}, false, toRideBigInt(123456789)},
{[]rideType{rideString("௦௧௨௩௪௫௬௭௮௯")}, false, toRideBigInt(123456789)},
{[]rideType{rideString("၀၁၂၃၄၅၆၇၈၉")}, false, toRideBigInt(123456789)},
{[]rideType{rideString("០១២៣៤៥៦៧៨៩")}, false, toRideBigInt(123456789)},
{[]rideType{rideString("0"), rideInt(4)}, false, newUnit(nil)},
{[]rideType{rideInt(0)}, false, newUnit(nil)},
{[]rideType{}, false, newUnit(nil)},
} {
r, err := stringToBigIntOpt(nil, test.args...)
if test.fail {
assert.Error(t, err)
} else {
require.NoError(t, err)
assert.True(t, test.r.eq(r), fmt.Sprintf("%s != %s", test.r, r))
}
t.Run(fmt.Sprintf("test_%d", i+1), func(t *testing.T) {
r, err := stringToBigIntOpt(nil, test.args...)
if test.fail {
assert.Error(t, err)
} else {
require.NoError(t, err)
assert.True(t, test.r.eq(r), fmt.Sprintf("%s != %s", test.r, r))
}
})
}
}

Expand Down
78 changes: 76 additions & 2 deletions pkg/ride/functions_strings.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
package ride

import (
"fmt"
"strconv"
"strings"
"unicode"
"unicode/utf16"
"unicode/utf8"

"github.com/ccoveille/go-safecast/v2"
"github.com/pkg/errors"

"github.com/wavesplatform/gowaves/pkg/proto"
Expand Down Expand Up @@ -302,10 +305,14 @@
if err != nil {
return nil, errors.Wrap(err, "parseInt")
}
i, err := strconv.ParseInt(string(s), 10, 64)
ns, err := normalizeDigits(string(s))
if err != nil {
return rideUnit{}, nil
return rideUnit{}, nil //nolint:nilerr // Suppress invalid digits string error, return unit instead.
}
i, err := strconv.ParseInt(ns, 10, 64)
if err != nil {
return rideUnit{}, nil //nolint:nilerr // Suppress conversion error, return unit instead.
}

Check warning

Code scanning / Semgrep OSS

Semgrep Finding: semgrep.rules.if-incorrect-nil-err-return Warning

WARNING: A local variable 'err' is checked for nil, but a different variable is returned.
Ensure that the returned variable is the one that was checked or properly wrapped!
return rideInt(i), nil
}

Expand All @@ -317,6 +324,73 @@
return extractValue(maybeInt)
}

func digitValueInRange(r rune, lo, hi, stride uint32) (byte, bool) {
v, err := safecast.Convert[uint32](r)
if err != nil {
return 0, false
}
if v < lo || v > hi {
return 0, false
}
if (v-lo)%stride != 0 {
return 0, false
}
return byte((v - lo) / stride % base10), true
}

func digitValue(r rune) (byte, bool) {
for _, rt := range unicode.Digit.R16 {
if v, ok := digitValueInRange(r, uint32(rt.Lo), uint32(rt.Hi), uint32(rt.Stride)); ok {
return v, true
}
}
for _, rt := range unicode.Digit.R32 {
Comment thread
nickeskov marked this conversation as resolved.
if v, ok := digitValueInRange(r, rt.Lo, rt.Hi, rt.Stride); ok {
return v, true
}
}
return 0, false
}

// isMathematicalStyledDigit returns true for Mathematical Alphanumeric Symbols like 𝟎-𝟗, 𝟘-𝟡, 𝟢-𝟫, 𝟬-𝟵, 𝟶-𝟿.
func isMathematicalStyledDigit(r rune) bool {
return '\U0001D7CE' <= r && r <= '\U0001D7FF'
}

func normalizeDigits(s string) (string, error) {
var b strings.Builder
for i, r := range s {
switch {
case r == '+' || r == '-':
// Sign is allowed only at the beginning.
if i != 0 {
return "", fmt.Errorf("unexpected sign %q", r)
}
b.WriteRune(r)
case '0' <= r && r <= '9':
b.WriteRune(r)
case unicode.IsDigit(r):
// Go's `unicode.IsDigit` reports more Unicode characters as digits than Java's implementation.
// Based on tests, mathematically styled digits are not treated as digits in the Scala node,
// so we filter them out here to preserve compatibility.
//
// This behavior may change in future versions of either platform, so compatibility should
// be rechecked when updating Go or Scala versions.
if isMathematicalStyledDigit(r) {
return "", fmt.Errorf("unsupported styled digit %q", r)
}
v, ok := digitValue(r)
if !ok {
return "", fmt.Errorf("unsupported digit %q", r)
}
b.WriteByte('0' + v)
default:
return "", fmt.Errorf("invalid character %q", r)
}
}
return b.String(), nil
}

func lastIndexOfSubstring(_ environment, args ...rideType) (rideType, error) {
s1, s2, err := twoStringsArgs(args)
if err != nil {
Expand Down
66 changes: 58 additions & 8 deletions pkg/ride/functions_strings_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -415,7 +415,7 @@ func BenchmarkSplitString4C(b *testing.B) {
}

func TestParseInt(t *testing.T) {
for _, test := range []struct {
for i, test := range []struct {
args []rideType
fail bool
r rideType
Expand All @@ -427,19 +427,49 @@ func TestParseInt(t *testing.T) {
{[]rideType{rideString("")}, false, rideUnit{}},
{[]rideType{rideString("123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890")}, false, rideUnit{}},
{[]rideType{rideString("abc")}, false, rideUnit{}},
{[]rideType{rideString("⓪①②③④⑤⑥⑦⑧⑨⑩⑪⑫⑬⑭⑮⑯⑰⑱⑲⑳")}, false, rideUnit{}},
{[]rideType{rideString("ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩ")}, false, rideUnit{}},
{[]rideType{rideString("ⅰⅱⅲⅳⅴⅵⅶⅷⅸⅹ")}, false, rideUnit{}},
{[]rideType{rideString("⁰¹²³⁴⁵⁶⁷⁸⁹")}, false, rideUnit{}},
{[]rideType{rideString("₀₁₂₃₄₅₆₇₈₉")}, false, rideUnit{}},
{[]rideType{rideString("𝟎𝟏𝟐𝟑𝟒𝟓𝟔𝟕𝟖𝟗")}, false, rideUnit{}},
{[]rideType{rideString("𝟘𝟙𝟚𝟛𝟜𝟝𝟞𝟟𝟠𝟡")}, false, rideUnit{}},
{[]rideType{rideString("𝟢𝟣𝟤𝟥𝟦𝟧𝟨𝟩𝟪𝟫")}, false, rideUnit{}},
{[]rideType{rideString("𝟬𝟭𝟮𝟯𝟰𝟱𝟲𝟳𝟴𝟵")}, false, rideUnit{}},
{[]rideType{rideString("𝟶𝟷𝟸𝟹𝟺𝟻𝟼𝟽𝟾𝟿")}, false, rideUnit{}},
{[]rideType{rideString("０１２３４５６７８９")}, false, rideInt(123456789)},
{[]rideType{rideString("٠١٢٣٤٥٦٧٨٩")}, false, rideInt(123456789)},
{[]rideType{rideString("۰۱۲۳۴۵۶۷۸۹")}, false, rideInt(123456789)},
{[]rideType{rideString("०१२३४५६७८९")}, false, rideInt(123456789)},
{[]rideType{rideString("০১২৩৪৫৬৭৮৯")}, false, rideInt(123456789)},
{[]rideType{rideString("๐๑๒๓๔๕๖๗๘๙")}, false, rideInt(123456789)},
{[]rideType{rideString("௦௧௨௩௪௫௬௭௮௯")}, false, rideInt(123456789)},
{[]rideType{rideString("၀၁၂၃၄၅၆၇၈၉")}, false, rideInt(123456789)},
{[]rideType{rideString("០១២៣៤៥៦៧៨៩")}, false, rideInt(123456789)},
{[]rideType{rideString("１２３")}, false, rideInt(123)},
{[]rideType{rideString("１００")}, false, rideInt(100)},
{[]rideType{rideString("-１２３")}, false, rideInt(-123)},
{[]rideType{rideString("+１２３")}, false, rideInt(123)},
{[]rideType{rideString("0１２３456")}, false, rideInt(123456)},
{[]rideType{rideString("１２٣٤")}, false, rideInt(1234)},
{[]rideType{rideString("١٢٣")}, false, rideInt(123)},
{[]rideType{rideString("-١٢٣")}, false, rideInt(-123)},
{[]rideType{rideString("-۱۲۳")}, false, rideInt(-123)},
{[]rideType{rideString("abc"), rideInt(0)}, true, nil},
{[]rideType{rideUnit{}}, true, nil},
{[]rideType{rideInt(1), rideString("x")}, true, nil},
{[]rideType{rideInt(1)}, true, nil},
{[]rideType{}, true, nil},
} {
r, err := parseInt(nil, test.args...)
if test.fail {
assert.Error(t, err)
} else {
require.NoError(t, err)
assert.Equal(t, test.r, r)
}
t.Run(fmt.Sprintf("test_%d", i+1), func(t *testing.T) {
r, err := parseInt(nil, test.args...)
if test.fail {
assert.Error(t, err)
} else {
require.NoError(t, err)
assert.Equal(t, test.r, r)
}
})
}
}

Expand Down Expand Up @@ -675,3 +705,23 @@ func TestContains(t *testing.T) {
}
}
}

func TestDigitsStringNormalization(t *testing.T) {
for i, test := range []struct {
s string
e string
}{
{"", ""},
{"123", "123"},
{"１２３", "123"},
{"-１２３", "-123"},
{"+１２３", "+123"},
{"0１２３456", "0123456"},
} {
t.Run(fmt.Sprintf("test_%d", i+1), func(t *testing.T) {
n, err := normalizeDigits(test.s)
require.NoError(t, err)
require.Equal(t, test.e, n)
})
}
}
6 changes: 4 additions & 2 deletions pkg/state/appender.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import (
"log/slog"

"github.com/ccoveille/go-safecast/v2"
"github.com/mr-tron/base58/base58"
"github.com/mr-tron/base58"
"github.com/pkg/errors"

"github.com/wavesplatform/gowaves/pkg/crypto"
Expand Down Expand Up @@ -716,7 +716,9 @@ func (a *txAppender) appendTxs(
txSnap, errAppendTx := a.appendTx(tx, appendTxArgs)
if errAppendTx != nil { // TODO: check error type for elided tx
if !isBlockWithChallenge {
return proto.BlockSnapshot{}, crypto.Digest{}, errAppendTx
return proto.BlockSnapshot{}, crypto.Digest{},
errors.Wrapf(errAppendTx, "failed to append tx %q at height %d", base58.Encode(txID),
blockInfo.Height)
}
slog.Debug("Elided tx detected", slog.String("ID", base58.Encode(txID)), logging.Error(errAppendTx))
txSnap = txSnapshot{
Expand Down
Loading