Skip to content

Commit aed1c22

Browse files
author
fredbi
authored
purge: added index compaction after the index is loaded (#743)
* purge: added index compaction after the index is loaded Signed-off-by: Frederic BIDON <frederic@oneconcern.com> * pebble: fixed edge case with invalid ranges Signed-off-by: Frederic BIDON <frederic@oneconcern.com> Signed-off-by: Frederic BIDON <frederic@oneconcern.com>
1 parent 4e6b4dd commit aed1c22

6 files changed

Lines changed: 45 additions & 10 deletions

File tree

‎go.mod‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ require (
1212
github.com/cenkalti/backoff/v4 v4.2.0
1313
github.com/cespare/xxhash/v2 v2.2.0 // indirect
1414
github.com/cockroachdb/errors v1.9.0 // indirect
15-
github.com/cockroachdb/pebble v0.0.0-20221229212011-811a8c0e741b
15+
github.com/cockroachdb/pebble v0.0.0-20230104192001-3d9c6101a3a1
1616
github.com/davecgh/go-spew v1.1.1
1717
github.com/dgraph-io/badger/v3 v3.2103.5
1818
github.com/docker/go-units v0.5.0
@@ -27,7 +27,7 @@ require (
2727
github.com/jacobsa/daemonize v0.0.0-20160101105449-e460293e890f
2828
github.com/jacobsa/fuse v0.0.0-20220531202254-21122235c77a
2929
github.com/karrick/godirwalk v1.17.0
30-
github.com/klauspost/compress v1.15.13 // indirect
30+
github.com/klauspost/compress v1.15.14 // indirect
3131
github.com/kr/pretty v0.3.1 // indirect
3232
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1
3333
github.com/nightlyone/lockfile v1.0.0
@@ -47,9 +47,10 @@ require (
4747
go.uber.org/atomic v1.10.0 // indirect
4848
go.uber.org/goleak v1.2.0
4949
go.uber.org/zap v1.24.0
50-
golang.org/x/exp v0.0.0-20221230185412-738e83a70c30 // indirect
50+
golang.org/x/exp v0.0.0-20230105000112-eab7a2c85304 // indirect
5151
golang.org/x/sync v0.1.0
5252
golang.org/x/sys v0.4.0
53+
golang.org/x/text v0.6.0 // indirect
5354
google.golang.org/api v0.106.0
5455
gopkg.in/yaml.v2 v2.4.0
5556
gotest.tools v2.2.0+incompatible

‎go.sum‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -600,8 +600,8 @@ github.com/cockroachdb/errors v1.9.0/go.mod h1:vaNcEYYqbIqB5JhKBhFV9CneUqeuEbB2O
600600
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
601601
github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f h1:6jduT9Hfc0njg5jJ1DdKCFPdMBrp/mdZfCpa5h+WM74=
602602
github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
603-
github.com/cockroachdb/pebble v0.0.0-20221229212011-811a8c0e741b h1:6TtBvyITBhpkL8EvEEZdS9om8egSxP/mUML01aM3pyY=
604-
github.com/cockroachdb/pebble v0.0.0-20221229212011-811a8c0e741b/go.mod h1:JsehdjcR1QgLZkqBeYrbVdE3cdxbdrycA/PN+Cg+RNw=
603+
github.com/cockroachdb/pebble v0.0.0-20230104192001-3d9c6101a3a1 h1:kYBdhwvFKV10NSq13QN8iCSwi7NLOGnqfWpf+FaYiA8=
604+
github.com/cockroachdb/pebble v0.0.0-20230104192001-3d9c6101a3a1/go.mod h1:JsehdjcR1QgLZkqBeYrbVdE3cdxbdrycA/PN+Cg+RNw=
605605
github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
606606
github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ=
607607
github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
@@ -1247,8 +1247,8 @@ github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e
12471247
github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
12481248
github.com/klauspost/compress v1.15.10/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
12491249
github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
1250-
github.com/klauspost/compress v1.15.13 h1:NFn1Wr8cfnenSJSA46lLq4wHCcBzKTSjnBIexDMMOV0=
1251-
github.com/klauspost/compress v1.15.13/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
1250+
github.com/klauspost/compress v1.15.14 h1:i7WCKDToww0wA+9qrUZ1xOjp218vfFo3nTU6UHp+gOc=
1251+
github.com/klauspost/compress v1.15.14/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
12521252
github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
12531253
github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
12541254
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
@@ -1860,8 +1860,8 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH
18601860
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
18611861
golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
18621862
golang.org/x/exp v0.0.0-20211216164055-b2b84827b756/go.mod h1:b9TAUYHmRtqA6klRHApnXMnj+OyLce4yF5cZCUbk2ps=
1863-
golang.org/x/exp v0.0.0-20221230185412-738e83a70c30 h1:m9O6OTJ627iFnN2JIWfdqlZCzneRO6EEBsHXI25P8ws=
1864-
golang.org/x/exp v0.0.0-20221230185412-738e83a70c30/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
1863+
golang.org/x/exp v0.0.0-20230105000112-eab7a2c85304 h1:YUqj+XKtfrn3kXjFIiZ8jwKROD7ioAOOHUuo3ZZ2opc=
1864+
golang.org/x/exp v0.0.0-20230105000112-eab7a2c85304/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
18651865
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
18661866
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
18671867
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
@@ -2201,8 +2201,9 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
22012201
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
22022202
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
22032203
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
2204-
golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
22052204
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
2205+
golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
2206+
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
22062207
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
22072208
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
22082209
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

‎pkg/core/kv.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ type (
2020
SetIfNotExists([]byte, []byte) error
2121
// AllKeys returns an iterator over all keys in the DB
2222
AllKeys() kvIterator
23+
// Compact the KV
24+
Compact() error
2325
}
2426

2527
// kvIterator provides a simplified abstraction to a KV iterator

‎pkg/core/purge.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -847,6 +847,10 @@ func copyIndexChunks(ctx context.Context, db kvStore, indexStore storage.Store,
847847
return indexTime, numKeys, lastIndex, fmt.Errorf("copy index chunks: %w", err)
848848
}
849849
logger.Debug("loaded all index chunks", zap.Uint64("num_keys", numKeys), zap.Uint64("last_chunk", lastIndex))
850+
if err := db.Compact(); err != nil {
851+
return indexTime, numKeys, lastIndex, err
852+
}
853+
logger.Debug("index compacted")
850854

851855
return indexTime, numKeys, lastIndex, nil
852856
}

‎pkg/core/purge_badger.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,10 @@ func (kv *kvBadger) SetIfNotExists(key, value []byte) error {
132132

133133
}
134134

135+
func (kv *kvBadger) Compact() error {
136+
return kv.DB.Flatten(100)
137+
}
138+
135139
func (i *kvBadgerIterator) Next() bool {
136140
if i.isFirst {
137141
i.iterator.Rewind()

‎pkg/core/purge_pebble.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,19 @@ func (kv *kvPebble) Drop() error {
3232
}()
3333

3434
start, end := iterator.RangeBounds()
35+
if pebble.DefaultComparer.Compare(start, end) >= 0 {
36+
return nil
37+
}
38+
3539
if err := kv.DB.DeleteRange(start, end, &pebble.WriteOptions{Sync: false}); err != nil {
3640
return err
3741
}
3842

43+
// as DeleteRange excludes the upper bound
44+
if err := kv.DB.Delete(end, &pebble.WriteOptions{Sync: false}); err != nil && !errors.Is(err, pebble.ErrNotFound) {
45+
return err
46+
}
47+
3948
return nil
4049
}
4150

@@ -90,6 +99,20 @@ func (kv *kvPebble) SetIfNotExists(key, value []byte) error {
9099
return kv.Merge(key, value, &pebble.WriteOptions{Sync: false}) // skip new value
91100
}
92101

102+
func (kv *kvPebble) Compact() error {
103+
iterator := kv.DB.NewIter(nil)
104+
start, end := iterator.RangeBounds()
105+
defer func() {
106+
_ = iterator.Close()
107+
}()
108+
109+
if pebble.DefaultComparer.Compare(start, end) >= 0 {
110+
return nil
111+
}
112+
113+
return kv.DB.Compact(start, end, true)
114+
}
115+
93116
func (i *kvPebbleIterator) Next() bool {
94117
if i.isFirst {
95118
_ = i.iterator.First()

0 commit comments

Comments
 (0)