Skip to content

Commit 8960e68

Browse files
mvachmvach
committed
fix: log blocked users without unicode escaping
1 parent b147a47 commit 8960e68

4 files changed

Lines changed: 72 additions & 42 deletions

File tree

‎csv/person_data.go‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"ctRestClient/config"
55
"ctRestClient/data_provider"
66
"ctRestClient/logger"
7+
"ctRestClient/utils"
78
"encoding/json"
89
"fmt"
910
)
@@ -36,7 +37,19 @@ func NewPersonData(
3637
}
3738
if isBlocked {
3839
blockCount++
39-
logger.Info(fmt.Sprintf(" -> %s %s will not be added to csv file", personJson["firstName"], personJson["lastName"]))
40+
41+
firstname, err := utils.UnescapeUnicodeCharacters(personJson["firstName"])
42+
if err != nil {
43+
logger.Error(fmt.Sprintf(" failed to unescape unicode characters in firstName: '%v'", err))
44+
firstname = personJson["firstName"]
45+
}
46+
lastname, err := utils.UnescapeUnicodeCharacters(personJson["lastName"])
47+
if err != nil {
48+
logger.Error(fmt.Sprintf(" failed to unescape unicode characters in lastName: '%v'", err))
49+
lastname = personJson["lastName"]
50+
}
51+
52+
logger.Info(fmt.Sprintf(" -> %s %s will not be added to csv file", firstname, lastname))
4053
continue
4154
}
4255

‎csv/person_data_test.go‎

Lines changed: 37 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -23,21 +23,22 @@ var _ = Describe("PersonData", func() {
2323
fileDataProvider *data_providerfakes.FakeFileDataProvider
2424
blocklistsDataProvider *data_providerfakes.FakeBlockListDataProvider
2525
logger *loggerfakes.FakeLogger
26+
person1, person2 string
2627
)
2728

2829
BeforeEach(func() {
29-
person1 := `{
30-
"id": 1,
31-
"firstName": "foo_firstname",
32-
"lastName": "foo_lastname",
30+
person1 = `{
31+
"id": 1,
32+
"firstName": "foo_firstname",
33+
"lastName": "foo_lastname",
3334
"height": 2.75
34-
}`
35-
person2 := `{
36-
"id": 2,
37-
"firstName": "bar_firstname",
38-
"lastName": "bar_lastname",
35+
}`
36+
person2 = `{
37+
"id": 2,
38+
"firstName": "bar_firstname",
39+
"lastName": "bar_lastname",
3940
"height": 1.0
40-
}`
41+
}`
4142
persons = []json.RawMessage{json.RawMessage(person1), json.RawMessage(person2)}
4243
fileDataProvider = &data_providerfakes.FakeFileDataProvider{}
4344
blocklistsDataProvider = &data_providerfakes.FakeBlockListDataProvider{}
@@ -69,33 +70,49 @@ var _ = Describe("PersonData", func() {
6970
blocklistsDataProvider.IsBlockedReturnsOnCall(0, true, nil)
7071
blocklistsDataProvider.IsBlockedReturnsOnCall(1, false, nil)
7172

72-
group := config.Group{Name: "test", Fields: []config.Field{{FieldName: ptr("id")}, {FieldName: ptr("unknown")}, {FieldName: ptr("lastName")}}}
73+
group := config.Group{Name: "test", Fields: []config.Field{{FieldName: ptr("id")}, {FieldName: ptr("firstName")}, {FieldName: ptr("lastName")}}}
7374
data, err := csv.NewPersonData(persons, group, fileDataProvider, blocklistsDataProvider, logger)
7475
Expect(err).NotTo(HaveOccurred())
7576

7677
Expect(logger.InfoArgsForCall(0)).To(Equal(" -> \"foo_firstname\" \"foo_lastname\" will not be added to csv file"))
77-
Expect(logger.WarnArgsForCall(0)).To(Equal(" Field 'unknown' does not exist"))
7878

79-
Expect(data.Header()).To(Equal([]string{"id", "unknown", "lastName"}))
79+
Expect(data.Header()).To(Equal([]string{"id", "firstName", "lastName"}))
8080
Expect(data.Records()).To(HaveLen(1))
81-
Expect(data.Records()[0]).To(Equal([]string{"2", "", "bar_lastname"}))
81+
Expect(data.Records()[0]).To(Equal([]string{"2", "bar_firstname", "bar_lastname"}))
82+
})
83+
84+
It("logs blocked persons with special characters", func() {
85+
person1 = `{
86+
"id": 1,
87+
"firstName": "\u00E4\u00FC\u00F6\u00DF_firstname",
88+
"lastName": "\u00C4\u00D6\u00DC\u00DF_lastname"
89+
}`
90+
persons = []json.RawMessage{json.RawMessage(person1), json.RawMessage(person2)}
91+
92+
blocklistsDataProvider.IsBlockedReturnsOnCall(0, true, nil)
93+
blocklistsDataProvider.IsBlockedReturnsOnCall(1, false, nil)
94+
95+
group := config.Group{Name: "test", Fields: []config.Field{{FieldName: ptr("id")}, {FieldName: ptr("firstName")}, {FieldName: ptr("lastName")}}}
96+
_, err := csv.NewPersonData(persons, group, fileDataProvider, blocklistsDataProvider, logger)
97+
Expect(err).NotTo(HaveOccurred())
98+
99+
Expect(logger.InfoArgsForCall(0)).To(Equal(" -> \"äüöß_firstname\" \"ÄÖÜß_lastname\" will not be added to csv file"))
82100
})
83101

84102
It("does not skip persons if an error occurs while checking blocklists", func() {
85103
blocklistsDataProvider.IsBlockedReturnsOnCall(0, false, nil)
86104
blocklistsDataProvider.IsBlockedReturnsOnCall(1, false, errors.New("boom"))
87105

88-
group := config.Group{Name: "test", Fields: []config.Field{{FieldName: ptr("id")}, {FieldName: ptr("unknown")}, {FieldName: ptr("lastName")}}}
106+
group := config.Group{Name: "test", Fields: []config.Field{{FieldName: ptr("id")}, {FieldName: ptr("firstName")}, {FieldName: ptr("lastName")}}}
89107
data, err := csv.NewPersonData(persons, group, fileDataProvider, blocklistsDataProvider, logger)
90108
Expect(err).NotTo(HaveOccurred())
91109

92110
Expect(logger.ErrorArgsForCall(0)).To(Equal(" failed to check if person is blocked: 'boom'"))
93-
Expect(logger.WarnArgsForCall(0)).To(Equal(" Field 'unknown' does not exist"))
94111

95-
Expect(data.Header()).To(Equal([]string{"id", "unknown", "lastName"}))
112+
Expect(data.Header()).To(Equal([]string{"id", "firstName", "lastName"}))
96113
Expect(data.Records()).To(HaveLen(2))
97-
Expect(data.Records()[0]).To(Equal([]string{"1", "", "foo_lastname"}))
98-
Expect(data.Records()[1]).To(Equal([]string{"2", "", "bar_lastname"}))
114+
Expect(data.Records()[0]).To(Equal([]string{"1", "foo_firstname", "foo_lastname"}))
115+
Expect(data.Records()[1]).To(Equal([]string{"2", "bar_firstname", "bar_lastname"}))
99116
})
100117

101118
It("sets unknown fields to empty string", func() {
@@ -115,7 +132,7 @@ var _ = Describe("PersonData", func() {
115132
person := `{
116133
"id": 1,
117134
"date": null
118-
}`
135+
}`
119136
persons = []json.RawMessage{json.RawMessage(person)}
120137
group := config.Group{Fields: []config.Field{{FieldName: ptr("id")}, {FieldName: ptr("date")}}}
121138
data, err := csv.NewPersonData(persons, group, fileDataProvider, blocklistsDataProvider, logger)
@@ -156,14 +173,6 @@ var _ = Describe("PersonData", func() {
156173
Expect(data.Records()[0]).To(Equal([]string{"1", "true"}))
157174
})
158175

159-
It("sets unknown fields to empty string", func() {
160-
group := config.Group{Fields: []config.Field{{FieldName: ptr("id")}, {FieldName: ptr("unknown")}}}
161-
_, err := csv.NewPersonData(persons, group, fileDataProvider, blocklistsDataProvider, logger)
162-
Expect(err).NotTo(HaveOccurred())
163-
164-
Expect(logger.WarnArgsForCall(0)).To(Equal(" Field 'unknown' does not exist"))
165-
})
166-
167176
It("returns mapped data for string key fields", func() {
168177
person1 := `{
169178
"id": 1,

‎data_provider/blocklist_data_provider.go‎

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"bytes"
55
"ctRestClient/config"
66
"ctRestClient/logger"
7+
"ctRestClient/utils"
78
"encoding/json"
89
"errors"
910
"fmt"
@@ -60,7 +61,7 @@ func (bp *blockListDataProvider) IsBlocked(personJson map[string]json.RawMessage
6061
} else {
6162

6263
// Unescape the json data that contains (\u00df and \u00fc instead of ß and ü) for later string comparison with data from the blocklist.
63-
personJsonValue, err := unescapeUnicodeCharacters(personJsonFieldValue)
64+
personJsonValue, err := utils.UnescapeUnicodeCharacters(personJsonFieldValue)
6465
if err != nil {
6566
return false, fmt.Errorf("failed to unescape unicode characters for field %s: %w", fieldName, err)
6667
}
@@ -145,15 +146,3 @@ func (bp *blockListDataProvider) BlockListExists(group config.Group) bool {
145146
return true
146147
}
147148

148-
func unescapeUnicodeCharacters(jsonRaw json.RawMessage) (json.RawMessage, error) {
149-
var temp interface{}
150-
if err := json.Unmarshal(jsonRaw, &temp); err != nil {
151-
return nil, fmt.Errorf("failed to unmarshal person json field")
152-
}
153-
154-
result, err := json.Marshal(temp)
155-
if err != nil {
156-
return nil, fmt.Errorf("failed to marshal person json field")
157-
}
158-
return result, nil
159-
}

‎utils/json.go‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
package utils
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
)
7+
8+
func UnescapeUnicodeCharacters(jsonRaw json.RawMessage) (json.RawMessage, error) {
9+
var temp interface{}
10+
if err := json.Unmarshal(jsonRaw, &temp); err != nil {
11+
return nil, fmt.Errorf("failed to unmarshal json raw message: %v", err)
12+
}
13+
14+
result, err := json.Marshal(temp)
15+
if err != nil {
16+
return nil, fmt.Errorf("failed to marshal json to raw message: %v", err)
17+
}
18+
return result, nil
19+
}

0 commit comments

Comments
 (0)