From 67de1d640db37daf0add6745fa7c5decf39885fb Mon Sep 17 00:00:00 2001 From: victorzhu999 <991212052@qq.com> Date: Thu, 2 Apr 2026 17:02:13 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8Dreslut=E8=AF=86?= =?UTF-8?q?=E5=88=ABslut=E7=9A=84=E8=AF=AF=E6=8A=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/bff-service/pkg/ahocorasick/api.go | 11 +- .../bff-service/pkg/ahocorasick/cloudflare.go | 135 ++++++++++++++++++ 2 files changed, 142 insertions(+), 4 deletions(-) diff --git a/internal/bff-service/pkg/ahocorasick/api.go b/internal/bff-service/pkg/ahocorasick/api.go index bc655b22c..59432c68e 100644 --- a/internal/bff-service/pkg/ahocorasick/api.go +++ b/internal/bff-service/pkg/ahocorasick/api.go @@ -196,16 +196,19 @@ func (a *acMgr) contentMatch(content string, dicts []DictConfig, returnFirstMatc for _, cfg := range dicts { dictKey := getDictKey(cfg.DictID, cfg.Version) if dict, exists := dictsCopy[dictKey]; exists { - matches := dict.Matcher.MatchThreadSafe(contentBytes) + matches := dict.Matcher.MatchThreadSafeWithPos(contentBytes) if len(matches) > 0 { - for _, idx := range matches { + for _, match := range matches { + if !isWordBoundaryMatch(contentBytes, match.Start, match.End, match.WordBytes) { + continue + } matchResult := MatchResult{ DictCfg: DictConfig{ DictID: dict.DictID, Version: dict.Version, }, - WordIndex: idx, - Word: dict.Matcher.getOriginalWord(idx), + WordIndex: match.Index, + Word: dict.Matcher.getOriginalWord(match.Index), Reply: dict.Reply, } results = append(results, matchResult) diff --git a/internal/bff-service/pkg/ahocorasick/cloudflare.go b/internal/bff-service/pkg/ahocorasick/cloudflare.go index 8632902b6..ba6875933 100644 --- a/internal/bff-service/pkg/ahocorasick/cloudflare.go +++ b/internal/bff-service/pkg/ahocorasick/cloudflare.go @@ -13,8 +13,17 @@ import ( "container/list" "sync" "sync/atomic" + "unicode" + "unicode/utf8" ) +type MatchPos struct { + Index int + Start int + End int + WordBytes []byte +} + // A node in the trie structure used to implement Aho-Corasick type node struct { root bool // true if this is the root @@ -233,6 +242,19 @@ func (m *Matcher) Match(in []byte) []int { }) } +// MatchWithPos is like Match but returns position information for each match +func (m *Matcher) MatchWithPos(in []byte) []MatchPos { + m.counter++ + + return matchWithPos(in, m.root, func(f *node) bool { + if f.counter != m.counter { + f.counter = m.counter + return true + } + return false + }) +} + // match is a core of matching logic. Accepts input byte slice, starting node // and a func to check whether should we include result into response or not func match(in []byte, n *node, unique func(f *node) bool) []int { @@ -274,6 +296,53 @@ func match(in []byte, n *node, unique func(f *node) bool) []int { return hits } +// matchWithPos is like match but returns position information +func matchWithPos(in []byte, n *node, unique func(f *node) bool) []MatchPos { + var hits []MatchPos + + for pos, b := range in { + c := int(b) + + if !n.root && n.child[c] == nil { + n = n.fails[c] + } + + if n.child[c] != nil { + f := n.child[c] + n = f + + if f.output { + if unique(f) { + start := pos - len(f.b) + 1 + hits = append(hits, MatchPos{ + Index: f.index, + Start: start, + End: pos + 1, + WordBytes: f.b, + }) + } + } + + for !f.suffix.root { + f = f.suffix + if unique(f) { + start := pos - len(f.b) + 1 + hits = append(hits, MatchPos{ + Index: f.index, + Start: start, + End: pos + 1, + WordBytes: f.b, + }) + } else { + break + } + } + } + } + + return hits +} + // MatchThreadSafe provides the same result as Match() but does it in a // thread-safe manner. Uses a sync.Pool of haystacks to track the uniqueness of // the result items. @@ -305,6 +374,72 @@ func (m *Matcher) MatchThreadSafe(in []byte) []int { return hits } +// MatchThreadSafeWithPos is like MatchThreadSafe but returns position information +func (m *Matcher) MatchThreadSafeWithPos(in []byte) []MatchPos { + var ( + heap map[int]uint64 + ) + + generation := atomic.AddUint64(&m.counter, 1) + n := m.root + // read the matcher's heap + item := m.heap.Get() + if item == nil { + heap = make(map[int]uint64, len(m.trie)) + } else { + heap = item.(map[int]uint64) + } + + hits := matchWithPos(in, n, func(f *node) bool { + g := heap[f.index] + if g != generation { + heap[f.index] = generation + return true + } + return false + }) + + m.heap.Put(heap) + return hits +} + +// isWordBoundary checks if a byte is a word boundary (not a letter or digit) +func isWordBoundary(b byte) bool { + return !unicode.IsLetter(rune(b)) && !unicode.IsDigit(rune(b)) +} + +// containsChinese checks if a byte slice contains Chinese characters +func containsChinese(b []byte) bool { + for len(b) > 0 { + r, size := utf8.DecodeRune(b) + if r == utf8.RuneError { + return false + } + if unicode.Is(unicode.Han, r) { + return true + } + b = b[size:] + } + return false +} + +// isWordBoundaryMatch checks if the match is at word boundaries (for non-Chinese words) +func isWordBoundaryMatch(content []byte, start, end int, wordBytes []byte) bool { + if containsChinese(wordBytes) { + return true + } + + if start > 0 && !isWordBoundary(content[start-1]) { + return false + } + + if end < len(content) && !isWordBoundary(content[end]) { + return false + } + + return true +} + // Contains returns true if any string matches. This can be faster // than Match() when you do not need to know which words matched. func (m *Matcher) Contains(in []byte) bool {