Skip to content

Commit 2f1bd07

Browse files
mvachmvach
committed
feat: Adding address blocklists
1 parent cbce869 commit 2f1bd07

15 files changed

Lines changed: 862 additions & 85 deletions

‎app/instances_processor.go‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,14 @@ import (
1313
)
1414

1515
type InstancesProcessor interface {
16-
Process(groupExporter GroupExporter, csvWriter csv.CSVFileWriter, rootDir string, personDataProvider csv.FileDataProvider, keepassCli KeepassCli) error
16+
Process(
17+
groupExporter GroupExporter,
18+
csvWriter csv.CSVFileWriter,
19+
rootDir string,
20+
personDataProvider csv.FileDataProvider,
21+
blocklistsDataProvider csv.BlockListDataProvider,
22+
keepassCli KeepassCli,
23+
) error
1724
}
1825

1926
type instancesProcessor struct {
@@ -31,7 +38,14 @@ func NewInstancesProcessor(
3138
}
3239
}
3340

34-
func (p instancesProcessor) Process(groupExporter GroupExporter, csvWriter csv.CSVFileWriter, rootDir string, personDataProvider csv.FileDataProvider, keepassCli KeepassCli) error {
41+
func (p instancesProcessor) Process(
42+
groupExporter GroupExporter,
43+
csvWriter csv.CSVFileWriter,
44+
rootDir string,
45+
fileDataProvider csv.FileDataProvider,
46+
blocklistsDataProvider csv.BlockListDataProvider,
47+
keepassCli KeepassCli,
48+
) error {
3549
for _, instance := range p.config.Instances {
3650

3751
p.logTitle(instance)
@@ -74,7 +88,11 @@ func (p instancesProcessor) Process(groupExporter GroupExporter, csvWriter csv.C
7488
p.logger.Info(fmt.Sprintf(" the group has %d persons", len(persons)))
7589
}
7690

77-
personData, err := csv.NewPersonData(persons, group.Fields, personDataProvider, p.logger)
91+
if blocklistsDataProvider.BlockListExists(group) {
92+
p.logger.Info(fmt.Sprintf(" using blocklist '%s.yml'", group.SanitizedGroupName()))
93+
}
94+
95+
personData, err := csv.NewPersonData(persons, group, fileDataProvider, blocklistsDataProvider, p.logger)
7896
if err != nil {
7997
p.logger.Error(fmt.Sprintf(" failed to extract persons: %v", err))
8098
continue

‎app/instances_processor_test.go‎

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,15 @@ import (
1717
var _ = Describe("InstanceProcessor", func() {
1818

1919
var (
20-
groupExporter *appfakes.FakeGroupExporter
21-
csvWriter *csvfakes.FakeCSVFileWriter
22-
logger *loggerfakes.FakeLogger
23-
keepassCli *appfakes.FakeKeepassCli
24-
personDataProvider *csvfakes.FakeFileDataProvider
25-
cfg config.Config
26-
instancesProcessor app.InstancesProcessor
27-
result []json.RawMessage
20+
groupExporter *appfakes.FakeGroupExporter
21+
csvWriter *csvfakes.FakeCSVFileWriter
22+
logger *loggerfakes.FakeLogger
23+
keepassCli *appfakes.FakeKeepassCli
24+
personDataProvider *csvfakes.FakeFileDataProvider
25+
blocklistsDataProvider *csvfakes.FakeBlockListDataProvider
26+
cfg config.Config
27+
instancesProcessor app.InstancesProcessor
28+
result []json.RawMessage
2829
)
2930

3031
BeforeEach(func() {
@@ -33,6 +34,7 @@ var _ = Describe("InstanceProcessor", func() {
3334
logger = &loggerfakes.FakeLogger{}
3435
keepassCli = &appfakes.FakeKeepassCli{}
3536
personDataProvider = &csvfakes.FakeFileDataProvider{}
37+
blocklistsDataProvider = &csvfakes.FakeBlockListDataProvider{}
3638

3739
cfg = config.Config{
3840
Instances: []config.Instance{
@@ -72,7 +74,7 @@ var _ = Describe("InstanceProcessor", func() {
7274
groupExporter.ExportGroupMembersReturns(result, nil)
7375
csvWriter.WriteReturns(nil)
7476

75-
err := instancesProcessor.Process(groupExporter, csvWriter, os.TempDir(), personDataProvider, keepassCli)
77+
err := instancesProcessor.Process(groupExporter, csvWriter, os.TempDir(), personDataProvider, blocklistsDataProvider, keepassCli)
7678
Expect(err).NotTo(HaveOccurred())
7779

7880
path, header, content := csvWriter.WriteArgsForCall(0)
@@ -100,7 +102,7 @@ var _ = Describe("InstanceProcessor", func() {
100102
keepassCli.GetPasswordReturns("", errors.New("booom"))
101103

102104
instancesProcessor = app.NewInstancesProcessor(cfg, logger)
103-
err := instancesProcessor.Process(groupExporter, csvWriter, os.TempDir(), personDataProvider, keepassCli)
105+
err := instancesProcessor.Process(groupExporter, csvWriter, os.TempDir(), personDataProvider, blocklistsDataProvider, keepassCli)
104106
Expect(err).NotTo(HaveOccurred())
105107

106108
message := logger.WarnArgsForCall(0)
@@ -112,7 +114,7 @@ var _ = Describe("InstanceProcessor", func() {
112114
groupExporter.ExportGroupMembersReturns(emptyGroupResult, nil)
113115
csvWriter.WriteReturns(nil)
114116

115-
err := instancesProcessor.Process(groupExporter, csvWriter, os.TempDir(), personDataProvider, keepassCli)
117+
err := instancesProcessor.Process(groupExporter, csvWriter, os.TempDir(), personDataProvider, blocklistsDataProvider, keepassCli)
116118
Expect(err).NotTo(HaveOccurred())
117119

118120
Expect(logger.InfoArgsForCall(2)).To(ContainSubstring("Processing instance 'foo'"))
@@ -124,7 +126,7 @@ var _ = Describe("InstanceProcessor", func() {
124126
groupExporter.ExportGroupMembersReturns(result, nil)
125127
csvWriter.WriteReturns(nil)
126128

127-
err := instancesProcessor.Process(groupExporter, csvWriter, os.TempDir(), personDataProvider, keepassCli)
129+
err := instancesProcessor.Process(groupExporter, csvWriter, os.TempDir(), personDataProvider, blocklistsDataProvider, keepassCli)
128130
Expect(err).NotTo(HaveOccurred())
129131

130132
Expect(logger.InfoArgsForCall(2)).To(ContainSubstring("Processing instance 'foo'"))
@@ -135,7 +137,7 @@ var _ = Describe("InstanceProcessor", func() {
135137
It("logs a warning for not active groups", func() {
136138
groupExporter.ExportGroupMembersReturns(nil, &app.GroupNotActiveError{GroupName: "foo_group"})
137139

138-
err := instancesProcessor.Process(groupExporter, csvWriter, os.TempDir(), personDataProvider, keepassCli)
140+
err := instancesProcessor.Process(groupExporter, csvWriter, os.TempDir(), personDataProvider, blocklistsDataProvider, keepassCli)
139141
Expect(err).NotTo(HaveOccurred())
140142

141143
Expect(logger.InfoArgsForCall(2)).To(ContainSubstring("Processing instance 'foo'"))
@@ -146,11 +148,11 @@ var _ = Describe("InstanceProcessor", func() {
146148
It("returns an error if person data export fails", func() {
147149
groupExporter.ExportGroupMembersReturns(nil, errors.New("boom"))
148150

149-
err := instancesProcessor.Process(groupExporter, csvWriter, os.TempDir(), personDataProvider, keepassCli)
151+
err := instancesProcessor.Process(groupExporter, csvWriter, os.TempDir(), personDataProvider, blocklistsDataProvider, keepassCli)
150152
Expect(err).NotTo(HaveOccurred())
151153

152154
Expect(logger.InfoArgsForCall(2)).To(ContainSubstring("Processing instance 'foo'"))
153-
Expect(logger.InfoArgsForCall(5)).To( Equal(" processing group 'foo_group'"))
155+
Expect(logger.InfoArgsForCall(5)).To(Equal(" processing group 'foo_group'"))
154156
Expect(logger.ErrorArgsForCall(0)).To(Equal(" failed to get person information: boom"))
155157
})
156158

@@ -160,7 +162,7 @@ var _ = Describe("InstanceProcessor", func() {
160162
groupExporter.ExportGroupMembersReturns(result, nil)
161163
csvWriter.WriteReturns(nil)
162164

163-
err := instancesProcessor.Process(groupExporter, csvWriter, os.TempDir(), personDataProvider, keepassCli)
165+
err := instancesProcessor.Process(groupExporter, csvWriter, os.TempDir(), personDataProvider, blocklistsDataProvider, keepassCli)
164166
Expect(err).ToNot(HaveOccurred())
165167

166168
Expect(logger.InfoArgsForCall(2)).To(ContainSubstring("Processing instance 'foo'"))

‎csv/blocklist_data_provider.go‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
package csv
2+
3+
import (
4+
"bytes"
5+
"ctRestClient/config"
6+
"ctRestClient/logger"
7+
"encoding/json"
8+
"errors"
9+
"fmt"
10+
"io/fs"
11+
"os"
12+
"path/filepath"
13+
14+
"gopkg.in/yaml.v3"
15+
)
16+
17+
//counterfeiter:generate . BlockListDataProvider
18+
type BlockListDataProvider interface {
19+
IsBlocked(personJson map[string]json.RawMessage, group config.Group) (bool, error)
20+
21+
BlockListExists(group config.Group) bool
22+
}
23+
24+
type addressData map[string]interface{}
25+
26+
type cacheEntry struct {
27+
data []addressData
28+
err error
29+
}
30+
31+
type blockListDataProvider struct {
32+
dataDir string
33+
dataCache map[string]cacheEntry
34+
logger logger.Logger
35+
}
36+
37+
func NewBlockListDataProvider(dataDir string, logger logger.Logger) BlockListDataProvider {
38+
return &blockListDataProvider{
39+
dataDir: dataDir,
40+
dataCache: make(map[string]cacheEntry),
41+
logger: logger,
42+
}
43+
}
44+
45+
func (bp *blockListDataProvider) IsBlocked(personJson map[string]json.RawMessage, group config.Group) (bool, error) {
46+
var entry cacheEntry
47+
48+
sanitizedGroupName := group.SanitizedGroupName()
49+
50+
entry, err := bp.loadBlocklist(sanitizedGroupName)
51+
if err != nil {
52+
return false, fmt.Errorf("failed to load blocklist for group %s: %w", sanitizedGroupName, err)
53+
}
54+
55+
for _, blockedAddress := range entry.data {
56+
matched := true
57+
58+
for fieldName, blockedFieldValue := range blockedAddress {
59+
if personJsonFieldValue, exists := personJson[fieldName]; !exists {
60+
bp.logger.Warn(fmt.Sprintf(" Ignoring blocklist element %v since field '%s' is not available in the person data", blockedAddress, fieldName))
61+
matched = false
62+
break
63+
} else {
64+
65+
// Unescape the json data that contains (\u00df and \u00fc instead of ß and ü) for later string comparison with data from the blocklist.
66+
personJsonValue, err := unescapeUnicodeCharacters(personJsonFieldValue)
67+
if err != nil {
68+
return false, fmt.Errorf("failed to unescape unicode characters for field %s: %w", fieldName, err)
69+
}
70+
71+
if !bytes.Equal([]byte(personJsonValue), blockedFieldValue.(json.RawMessage)) {
72+
matched = false
73+
break
74+
}
75+
}
76+
}
77+
if matched {
78+
return true, nil
79+
}
80+
}
81+
82+
return false, nil
83+
}
84+
85+
func (bp *blockListDataProvider) loadBlocklist(name string) (cacheEntry, error) {
86+
87+
if entry, ok := bp.dataCache[name]; ok {
88+
// ---- CACHE HIT ----
89+
return entry, entry.err
90+
}
91+
92+
path := filepath.Join(bp.dataDir, name+".yml")
93+
94+
yamlData, err := os.ReadFile(path)
95+
if errors.Is(err, fs.ErrNotExist) {
96+
entry := cacheEntry{}
97+
bp.dataCache[name] = entry
98+
return entry, nil
99+
}
100+
101+
if err != nil {
102+
entry := cacheEntry{err: err}
103+
bp.dataCache[name] = entry
104+
return entry, err
105+
}
106+
107+
if len(yamlData) == 0 {
108+
entry := cacheEntry{data: []addressData{}}
109+
bp.dataCache[name] = entry
110+
return entry, nil
111+
}
112+
113+
var blocked []addressData
114+
if err := yaml.Unmarshal(yamlData, &blocked); err != nil {
115+
entry := cacheEntry{err: err}
116+
bp.dataCache[name] = entry
117+
return entry, err
118+
}
119+
120+
for i, addr := range blocked {
121+
for key, val := range addr {
122+
jsonBytes, err := json.Marshal(val)
123+
if err != nil {
124+
return cacheEntry{}, fmt.Errorf("failed to marshal blocklist value for key %s: %w", key, err)
125+
}
126+
addr[key] = json.RawMessage(jsonBytes)
127+
}
128+
blocked[i] = addr
129+
}
130+
131+
entry := cacheEntry{data: blocked}
132+
bp.dataCache[name] = entry
133+
134+
return entry, nil
135+
}
136+
137+
func (bp *blockListDataProvider) BlockListExists(group config.Group) bool {
138+
blocklistFilePath := filepath.Join(bp.dataDir, group.SanitizedGroupName()+".yml")
139+
_, err := os.Stat(blocklistFilePath)
140+
if err != nil {
141+
if os.IsNotExist(err) {
142+
return false
143+
} else {
144+
bp.logger.Error(fmt.Sprintf(" failed to evaluate blocklist existence: %v", err))
145+
}
146+
}
147+
148+
return true
149+
}
150+
151+
func unescapeUnicodeCharacters(jsonRaw json.RawMessage) (json.RawMessage, error) {
152+
var temp interface{}
153+
if err := json.Unmarshal(jsonRaw, &temp); err != nil {
154+
return nil, fmt.Errorf("failed to unmarshal person json field")
155+
}
156+
157+
result, err := json.Marshal(temp)
158+
if err != nil {
159+
return nil, fmt.Errorf("failed to marshal person json field")
160+
}
161+
return result, nil
162+
}

0 commit comments

Comments
 (0)