Skip to content

Commit 87002d9

Browse files
GoSimplicityGoSimplicity
authored
Merge pull request #197 from GoSimplicity/demo
添加缓存插件布隆过滤 Former-commit-id: 961a1fb8badfce80de6ac6b8016ff5105dc71bc4
2 parents ed71452 + 490b1c7 commit 87002d9

4 files changed

Lines changed: 67 additions & 26 deletions

File tree

‎go.mod‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,13 @@ go 1.22.3
44

55
require (
66
github.com/IBM/sarama v1.43.2
7+
github.com/bits-and-blooms/bloom/v3 v3.7.0
78
github.com/bsm/redislock v0.9.4
89
github.com/bwmarrin/snowflake v0.3.0
910
github.com/casbin/casbin/v2 v2.93.0
1011
github.com/casbin/gorm-adapter/v3 v3.25.0
1112
github.com/dlclark/regexp2 v1.11.0
13+
github.com/elastic/go-elasticsearch/v8 v8.14.0
1214
github.com/gin-contrib/cors v1.7.2
1315
github.com/gin-gonic/gin v1.10.0
1416
github.com/go-sql-driver/mysql v1.8.1
@@ -19,7 +21,6 @@ require (
1921
github.com/robfig/cron/v3 v3.0.1
2022
github.com/spf13/pflag v1.0.5
2123
github.com/spf13/viper v1.18.2
22-
github.com/swaggo/swag v1.16.3
2324
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.933
2425
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/sms v1.0.933
2526
go.mongodb.org/mongo-driver v1.15.0
@@ -34,6 +35,7 @@ require (
3435

3536
require (
3637
filippo.io/edwards25519 v1.1.0 // indirect
38+
github.com/bits-and-blooms/bitset v1.13.0 // indirect
3739
github.com/casbin/govaluate v1.1.1 // indirect
3840
github.com/cespare/xxhash/v2 v2.2.0 // indirect
3941
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
@@ -43,7 +45,6 @@ require (
4345
github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
4446
github.com/eapache/queue v1.1.0 // indirect
4547
github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect
46-
github.com/elastic/go-elasticsearch/v8 v8.14.0 // indirect
4748
github.com/glebarez/go-sqlite v1.22.0 // indirect
4849
github.com/glebarez/sqlite v1.11.0 // indirect
4950
github.com/go-logr/logr v1.4.2 // indirect
@@ -77,6 +78,7 @@ require (
7778
go.opentelemetry.io/otel v1.28.0 // indirect
7879
go.opentelemetry.io/otel/metric v1.28.0 // indirect
7980
go.opentelemetry.io/otel/trace v1.28.0 // indirect
81+
golang.org/x/mod v0.18.0 // indirect
8082
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
8183
gorm.io/driver/postgres v1.5.7 // indirect
8284
gorm.io/driver/sqlserver v1.5.3 // indirect
@@ -88,18 +90,13 @@ require (
8890
)
8991

9092
require (
91-
github.com/KyleBanks/depth v1.2.1 // indirect
9293
github.com/bytedance/sonic v1.11.8 // indirect
9394
github.com/bytedance/sonic/loader v0.1.1 // indirect
9495
github.com/cloudwego/base64x v0.1.4 // indirect
9596
github.com/cloudwego/iasm v0.2.0 // indirect
9697
github.com/fsnotify/fsnotify v1.7.0 // indirect
9798
github.com/gabriel-vasile/mimetype v1.4.4 // indirect
9899
github.com/gin-contrib/sse v0.1.0 // indirect
99-
github.com/go-openapi/jsonpointer v0.21.0 // indirect
100-
github.com/go-openapi/jsonreference v0.21.0 // indirect
101-
github.com/go-openapi/spec v0.21.0 // indirect
102-
github.com/go-openapi/swag v0.23.0 // indirect
103100
github.com/go-playground/locales v0.14.1 // indirect
104101
github.com/go-playground/universal-translator v0.18.1 // indirect
105102
github.com/go-playground/validator/v10 v10.22.0 // indirect
@@ -108,12 +105,10 @@ require (
108105
github.com/hashicorp/hcl v1.0.0 // indirect
109106
github.com/jinzhu/inflection v1.0.0 // indirect
110107
github.com/jinzhu/now v1.1.5 // indirect
111-
github.com/josharian/intern v1.0.0 // indirect
112108
github.com/json-iterator/go v1.1.12 // indirect
113109
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
114110
github.com/leodido/go-urn v1.4.0 // indirect
115111
github.com/magiconair/properties v1.8.7 // indirect
116-
github.com/mailru/easyjson v0.7.7 // indirect
117112
github.com/mattn/go-isatty v0.0.20 // indirect
118113
github.com/mitchellh/mapstructure v1.5.0 // indirect
119114
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect

‎go.sum‎

Lines changed: 9 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,11 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaC
2626
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
2727
github.com/IBM/sarama v1.43.2 h1:HABeEqRUh32z8yzY2hGB/j8mHSzC/HA9zlEjqFNCzSw=
2828
github.com/IBM/sarama v1.43.2/go.mod h1:Kyo4WkF24Z+1nz7xeVUFWIuKVV8RS3wM8mkvPKMdXFQ=
29-
github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
30-
github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
29+
github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
30+
github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE=
31+
github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
32+
github.com/bits-and-blooms/bloom/v3 v3.7.0 h1:VfknkqV4xI+PsaDIsoHueyxVDZrfvMn56jeWUzvzdls=
33+
github.com/bits-and-blooms/bloom/v3 v3.7.0/go.mod h1:VKlUSvp0lFIYqxJjzdnSsZEw4iHb1kOL2tfHTgyJBHg=
3134
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
3235
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
3336
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
@@ -98,14 +101,6 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
98101
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
99102
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
100103
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
101-
github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
102-
github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
103-
github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ=
104-
github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4=
105-
github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY=
106-
github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk=
107-
github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
108-
github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
109104
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
110105
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
111106
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
@@ -182,8 +177,6 @@ github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkr
182177
github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
183178
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
184179
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
185-
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
186-
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
187180
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
188181
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
189182
github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
@@ -204,8 +197,6 @@ github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8=
204197
github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
205198
github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
206199
github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
207-
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
208-
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
209200
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
210201
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
211202
github.com/microsoft/go-mssqldb v1.6.0/go.mod h1:00mDtPbeQCRGC1HwOOR5K/gr30P1NcEG0vx6Kbv2aJU=
@@ -273,14 +264,14 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
273264
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
274265
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
275266
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
276-
github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg=
277-
github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk=
278267
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.933 h1:U9cMP5WSO7GNZoQ0Br2MlzG31KU+WSeyCaXeUNV4Mow=
279268
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.933/go.mod h1:r5r4xbfxSaeR04b166HGsBa/R4U3SueirEUpXGuw+Q0=
280269
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/sms v1.0.933 h1:JVkDpW7y/Rl+FYmO6kuYv9Q12KJ3AAMrG0QqSMzwIHQ=
281270
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/sms v1.0.933/go.mod h1:nJzJ+3+BDMeYd34uziUJK5KMv2QEHgYJkGLxUpVMNc0=
282271
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
283272
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
273+
github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg=
274+
github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
284275
github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
285276
github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
286277
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
@@ -299,6 +290,8 @@ go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo=
299290
go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4=
300291
go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q=
301292
go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s=
293+
go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8=
294+
go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E=
302295
go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g=
303296
go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI=
304297
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=

‎pkg/cachebloom/cachebloom.go‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package cachebloom
2+
3+
import (
4+
"context"
5+
"errors"
6+
"github.com/bits-and-blooms/bloom/v3"
7+
"github.com/redis/go-redis/v9"
8+
"time"
9+
)
10+
11+
// CacheBloom 是一个包含布隆过滤器和 Redis 客户端的结构体
12+
type CacheBloom struct {
13+
bf *bloom.BloomFilter
14+
client *redis.Client
15+
}
16+
17+
// NewCacheBloom 创建并初始化一个 CacheBloom 实例
18+
func NewCacheBloom(client *redis.Client, filterSize uint, falsePositiveRate float64) *CacheBloom {
19+
return &CacheBloom{
20+
bf: bloom.NewWithEstimates(filterSize, falsePositiveRate),
21+
client: client,
22+
}
23+
}
24+
25+
// QueryData 查询数据，如果缓存和布隆过滤器都没有命中，则查询数据库
26+
func (cb *CacheBloom) QueryData(ctx context.Context, key string, queryDB func(string) string, ttl time.Duration) (string, error) {
27+
// 检查布隆过滤器
28+
if !cb.bf.TestString(key) {
29+
return "", nil // 数据不存在，直接返回空结果
30+
}
31+
// 检查缓存
32+
data, err := cb.client.Get(ctx, key).Result()
33+
if err != nil {
34+
if errors.Is(err, redis.Nil) {
35+
// 缓存中没有数据，查询数据库
36+
data = queryDB(key)
37+
if data != "" {
38+
if er := cb.client.Set(ctx, key, data, ttl).Err(); er != nil {
39+
return "", er // 返回设置缓存时的错误
40+
}
41+
cb.bf.AddString(key) // 更新布隆过滤器
42+
} else {
43+
if er := cb.client.Set(ctx, key, "", ttl).Err(); er != nil {
44+
return "", er // 返回设置缓存空对象时的错误
45+
}
46+
cb.bf.AddString(key) // 更新布隆过滤器
47+
}
48+
} else {
49+
return "", err // 返回Redis操作时发生的其他错误
50+
}
51+
}
52+
return data, nil
53+
}

‎pkg/limiterp/limit.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ func NewRedisSlidingWindowLimiter(cmd redis.Cmdable, interval time.Duration, rat
3131
}
3232
}
3333

34-
func (l limiter) Limit(ctx context.Context, key string) (bool, error) {
34+
func (l *limiter) Limit(ctx context.Context, key string) (bool, error) {
3535
return l.cmd.Eval(ctx, luaScript, []string{key},
3636
l.interval.Milliseconds(), l.rate, time.Now().UnixMilli()).Bool()
3737
}

0 commit comments

Comments
 (0)