Skip to content

Commit dfc6328

Browse files
tangentayou06
tangenta
and
you06
authored
(tidb-8.5) support redact key in logs (tikv#1612) (tikv#1633)
ref pingcap/tidb#59279 Signed-off-by: tangenta <tangenta@126.com> Co-authored-by: you06 <you1474600@gmail.com>
1 parent cc8b949 commit dfc6328

24 files changed

Lines changed: 217 additions & 126 deletions

File tree

‎error/error.go‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@
3535
package error
3636

3737
import (
38-
"encoding/hex"
3938
"fmt"
4039
"time"
4140

@@ -45,6 +44,7 @@ import (
4544
"github.com/pkg/errors"
4645
"github.com/tikv/client-go/v2/internal/logutil"
4746
"github.com/tikv/client-go/v2/util"
47+
"github.com/tikv/client-go/v2/util/redact"
4848
"go.uber.org/zap"
4949
)
5050

@@ -289,13 +289,13 @@ func (e *ErrAssertionFailed) Error() string {
289289
func (e *ErrLockOnlyIfExistsNoReturnValue) Error() string {
290290
return fmt.Sprintf("LockOnlyIfExists is set for Lock Context, but ReturnValues is not set, "+
291291
"StartTs is {%d}, ForUpdateTs is {%d}, one of lock keys is {%v}.",
292-
e.StartTS, e.ForUpdateTs, hex.EncodeToString(e.LockKey))
292+
e.StartTS, e.ForUpdateTs, redact.Key(e.LockKey))
293293
}
294294

295295
func (e *ErrLockOnlyIfExistsNoPrimaryKey) Error() string {
296296
return fmt.Sprintf("LockOnlyIfExists is set for Lock Context, but primary key of current transaction is not set, "+
297297
"StartTs is {%d}, ForUpdateTs is {%d}, one of lock keys is {%s}",
298-
e.StartTS, e.ForUpdateTs, hex.EncodeToString(e.LockKey))
298+
e.StartTS, e.ForUpdateTs, redact.Key(e.LockKey))
299299
}
300300

301301
// ExtractKeyErr extracts a KeyError.
@@ -307,6 +307,8 @@ func ExtractKeyErr(keyErr *kvrpcpb.KeyError) error {
307307
}
308308
}
309309

310+
redact.RedactKeyErrIfNecessary(keyErr)
311+
310312
if keyErr.Conflict != nil {
311313
return errors.WithStack(&ErrWriteConflict{WriteConflict: keyErr.GetConflict()})
312314
}

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ require (
1212
github.com/google/uuid v1.6.0
1313
github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
1414
github.com/opentracing/opentracing-go v1.2.0
15-
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c
15+
github.com/pingcap/errors v0.11.5-0.20241219054535-6b8c588c3122
1616
github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c
1717
github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989
1818
github.com/pingcap/kvproto v0.0.0-20240924080114-4a3e17f5e62d

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,8 @@ github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+
6868
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
6969
github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
7070
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
71-
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4=
72-
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg=
71+
github.com/pingcap/errors v0.11.5-0.20241219054535-6b8c588c3122 h1:jc1bYMk3a2uD0+yK6Y8sRDrqvRELf/u2foUu7IT40Dw=
72+
github.com/pingcap/errors v0.11.5-0.20241219054535-6b8c588c3122/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg=
7373
github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c h1:CgbKAHto5CQgWM9fSBIvaxsJHuGP0uM74HXtv3MyyGQ=
7474
github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew=
7575
github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E=

‎integration_tests/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ go 1.23
44

55
require (
66
github.com/ninedraft/israce v0.0.3
7-
github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f
7+
github.com/pingcap/errors v0.11.5-0.20241219054535-6b8c588c3122
88
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86
99
github.com/pingcap/kvproto v0.0.0-20241120022153-92b0414aeed8
1010
github.com/pingcap/tidb v1.1.0-beta.0.20250106104940-3ac9806e2a76

‎integration_tests/go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -352,8 +352,8 @@ github.com/pingcap/badger v1.5.1-0.20241015064302-38533b6cbf8d/go.mod h1:KiO2zum
352352
github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
353353
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
354354
github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
355-
github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f h1:FxA+NgsdHNOv+/hZGxUh8Gb3WuZqgqmxDwztEOiA1v4=
356-
github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg=
355+
github.com/pingcap/errors v0.11.5-0.20241219054535-6b8c588c3122 h1:jc1bYMk3a2uD0+yK6Y8sRDrqvRELf/u2foUu7IT40Dw=
356+
github.com/pingcap/errors v0.11.5-0.20241219054535-6b8c588c3122/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg=
357357
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE=
358358
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4=
359359
github.com/pingcap/fn v1.0.0 h1:CyA6AxcOZkQh52wIqYlAmaVmF6EvrcqFywP463pjA8g=

‎integration_tests/scan_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,11 +40,11 @@ import (
4040
"testing"
4141

4242
"github.com/stretchr/testify/suite"
43-
"github.com/tikv/client-go/v2/kv"
4443
"github.com/tikv/client-go/v2/tikv"
4544
"github.com/tikv/client-go/v2/txnkv"
4645
"github.com/tikv/client-go/v2/txnkv/transaction"
4746
"github.com/tikv/client-go/v2/txnkv/txnsnapshot"
47+
"github.com/tikv/client-go/v2/util/redact"
4848
)
4949

5050
var scanBatchSize = tikv.ConfigProbe{}.GetScanBatchSize()
@@ -105,7 +105,7 @@ func (s *testScanSuite) TestScan() {
105105
for i := 0; i < rowNum; i++ {
106106
k := scan.Key()
107107
expectedKey := s.makeKey(i)
108-
s.Equal(k, expectedKey, "i=%v,rowNum=%v,key=%v,val=%v,expected=%v,keyOnly=%v", i, rowNum, kv.StrKey(k), kv.StrKey(scan.Value()), kv.StrKey(expectedKey), keyOnly)
108+
s.Equal(k, expectedKey, "i=%v,rowNum=%v,key=%v,val=%v,expected=%v,keyOnly=%v", i, rowNum, redact.Key(k), redact.Key(scan.Value()), redact.Key(expectedKey), keyOnly)
109109
if !keyOnly {
110110
v := scan.Value()
111111
s.Equal(v, s.makeValue(i))

‎internal/apicodec/codec_v2.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package apicodec
33
import (
44
"bytes"
55
"encoding/binary"
6-
"encoding/hex"
76

87
"github.com/pingcap/kvproto/pkg/coprocessor"
98
"github.com/pingcap/kvproto/pkg/errorpb"
@@ -13,6 +12,7 @@ import (
1312
"github.com/pkg/errors"
1413
"github.com/tikv/client-go/v2/internal/logutil"
1514
"github.com/tikv/client-go/v2/tikvrpc"
15+
"github.com/tikv/client-go/v2/util/redact"
1616
"go.uber.org/zap"
1717
)
1818

@@ -698,8 +698,8 @@ func (c *codecV2) DecodeKey(encodedKey []byte) ([]byte, error) {
698698
// return out of bound error.
699699
if !bytes.HasPrefix(encodedKey, c.prefix) {
700700
logutil.BgLogger().Warn("key not in keyspace",
701-
zap.String("keyspacePrefix", hex.EncodeToString(c.prefix)),
702-
zap.String("key", hex.EncodeToString(encodedKey)),
701+
zap.String("keyspacePrefix", redact.Key(c.prefix)),
702+
zap.String("key", redact.Key(encodedKey)),
703703
zap.Stack("stack"))
704704
return nil, errKeyOutOfBound
705705
}

‎internal/locate/region_cache.go‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ package locate
3737
import (
3838
"bytes"
3939
"context"
40-
"encoding/hex"
4140
"fmt"
4241
"math/rand"
4342
"slices"
@@ -66,6 +65,7 @@ import (
6665
"github.com/tikv/client-go/v2/metrics"
6766
"github.com/tikv/client-go/v2/tikvrpc"
6867
"github.com/tikv/client-go/v2/util"
68+
"github.com/tikv/client-go/v2/util/redact"
6969
pd "github.com/tikv/pd/client"
7070
"go.uber.org/zap"
7171
"google.golang.org/grpc/codes"
@@ -1105,7 +1105,7 @@ func (l *KeyLocation) Contains(key []byte) bool {
11051105

11061106
// String implements fmt.Stringer interface.
11071107
func (l *KeyLocation) String() string {
1108-
return fmt.Sprintf("region %s,startKey:%s,endKey:%s", l.Region.String(), kv.StrKey(l.StartKey), kv.StrKey(l.EndKey))
1108+
return fmt.Sprintf("region %s,startKey:%s,endKey:%s", l.Region.String(), redact.Key(l.StartKey), redact.Key(l.EndKey))
11091109
}
11101110

11111111
// GetBucketVersion gets the bucket version of the region.
@@ -1160,7 +1160,7 @@ func (l *KeyLocation) LocateBucket(key []byte) *Bucket {
11601160
}
11611161
// unreachable
11621162
logutil.Logger(context.Background()).Info(
1163-
"Unreachable place", zap.String("KeyLocation", l.String()), zap.String("Key", hex.EncodeToString(key)))
1163+
"Unreachable place", zap.String("KeyLocation", l.String()), zap.String("Key", redact.Key(key)))
11641164
panic("Unreachable")
11651165
}
11661166

@@ -1526,8 +1526,8 @@ func (c *RegionCache) findRegionByKey(bo *retry.Backoffer, key []byte, isEndKey
15261526
if err != nil {
15271527
// ignore error and use old region info.
15281528
logutil.Logger(bo.GetCtx()).Error("load region failure",
1529-
zap.String("key", util.HexRegionKeyStr(key)), zap.Error(err),
1530-
zap.String("encode-key", util.HexRegionKeyStr(c.codec.EncodeRegionKey(key))))
1529+
zap.String("key", redact.Key(key)), zap.Error(err),
1530+
zap.String("encode-key", redact.Key(c.codec.EncodeRegionKey(key))))
15311531
} else {
15321532
logutil.Eventf(bo.GetCtx(), "load region %d from pd, due to need-reload", lr.GetID())
15331533
reloadOnAccess := flags&needReloadOnAccess > 0
@@ -1780,8 +1780,8 @@ func (c *RegionCache) BatchLoadRegionsWithKeyRange(bo *retry.Backoffer, startKey
17801780
}
17811781
if len(regions) == 0 {
17821782
err = errors.Errorf("PD returned no region, start_key: %q, end_key: %q, encode_start_key: %q, encode_end_key: %q",
1783-
util.HexRegionKeyStr(startKey), util.HexRegionKeyStr(endKey),
1784-
util.HexRegionKeyStr(c.codec.EncodeRegionKey(startKey)), util.HexRegionKeyStr(c.codec.EncodeRegionKey(endKey)))
1783+
redact.Key(startKey), redact.Key(endKey),
1784+
redact.Key(c.codec.EncodeRegionKey(startKey)), redact.Key(c.codec.EncodeRegionKey(endKey)))
17851785
return
17861786
}
17871787

@@ -2081,13 +2081,13 @@ func (c *RegionCache) loadRegion(bo *retry.Backoffer, key []byte, isEndKey bool,
20812081
if err != nil {
20822082
if apicodec.IsDecodeError(err) {
20832083
return nil, errors.Errorf("failed to decode region range key, key: %q, err: %v, encode_key: %q",
2084-
util.HexRegionKeyStr(key), err, util.HexRegionKey(c.codec.EncodeRegionKey(key)))
2084+
redact.Key(key), err, redact.KeyBytes(c.codec.EncodeRegionKey(key)))
20852085
}
2086-
backoffErr = errors.Errorf("loadRegion from PD failed, key: %q, err: %v", util.HexRegionKeyStr(key), err)
2086+
backoffErr = errors.Errorf("loadRegion from PD failed, key: %q, err: %v", redact.Key(key), err)
20872087
continue
20882088
}
20892089
if reg == nil || reg.Meta == nil {
2090-
backoffErr = errors.Errorf("region not found for key %q, encode_key: %q", util.HexRegionKeyStr(key), util.HexRegionKey(c.codec.EncodeRegionKey(key)))
2090+
backoffErr = errors.Errorf("region not found for key %q, encode_key: %q", redact.Key(key), redact.KeyBytes(c.codec.EncodeRegionKey(key)))
20912091
continue
20922092
}
20932093
if len(reg.Meta.Peers) == 0 {

‎internal/logutil/hex.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,13 +36,13 @@ package logutil
3636

3737
import (
3838
"bytes"
39-
"encoding/hex"
4039
"fmt"
4140
"io"
4241
"reflect"
4342
"strings"
4443

4544
"github.com/golang/protobuf/proto" //nolint:staticcheck
45+
"github.com/tikv/client-go/v2/util/redact"
4646
)
4747

4848
// Hex defines a fmt.Stringer for proto.Message.
@@ -68,7 +68,7 @@ func prettyPrint(w io.Writer, val reflect.Value) {
6868
case reflect.Slice:
6969
elemType := tp.Elem()
7070
if elemType.Kind() == reflect.Uint8 {
71-
fmt.Fprintf(w, "%s", hex.EncodeToString(val.Bytes()))
71+
fmt.Fprintf(w, "%s", redact.Key(val.Bytes()))
7272
} else {
7373
fmt.Fprintf(w, "%s", val.Interface())
7474
}

‎internal/mockstore/mocktikv/errors.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,10 @@
3535
package mocktikv
3636

3737
import (
38-
"encoding/hex"
3938
"fmt"
4039

4140
"github.com/pingcap/kvproto/pkg/kvrpcpb"
41+
"github.com/tikv/client-go/v2/util/redact"
4242
)
4343

4444
// ErrLocked is returned when trying to Read/Write on a locked key. Client should
@@ -98,7 +98,7 @@ type ErrAlreadyRollbacked struct {
9898
}
9999

100100
func (e *ErrAlreadyRollbacked) Error() string {
101-
return fmt.Sprintf("txn=%v on key=%s is already rolled back", e.startTS, hex.EncodeToString(e.key))
101+
return fmt.Sprintf("txn=%v on key=%s is already rolled back", e.startTS, redact.Key(e.key))
102102
}
103103

104104
// ErrConflict is returned when the commitTS of key in the DB is greater than startTS.
@@ -154,5 +154,5 @@ type ErrAssertionFailed struct {
154154

155155
func (e *ErrAssertionFailed) Error() string {
156156
return fmt.Sprintf("AssertionFailed { StartTS: %v, Key: %v, Assertion: %v, ExistingStartTS: %v, ExistingCommitTS: %v }",
157-
e.StartTS, hex.EncodeToString(e.Key), e.Assertion.String(), e.ExistingStartTS, e.ExistingCommitTS)
157+
e.StartTS, redact.Key(e.Key), e.Assertion.String(), e.ExistingStartTS, e.ExistingCommitTS)
158158
}

0 commit comments

Comments
 (0)