forked from finos/symphony-bdk-python
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathinput_tokenizer_test.py
More file actions
155 lines (106 loc) · 6.42 KB
/
Copy pathinput_tokenizer_test.py
File metadata and controls
155 lines (106 loc) · 6.42 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
from symphony.bdk.core.activity.parsing.message_entities import Cashtag, Hashtag, Mention
from symphony.bdk.core.activity.parsing.input_tokenizer import InputTokenizer
from symphony.bdk.gen.agent_model.v4_message import V4Message
def test_empty_content():
under_test = build_tokenizer("")
assert under_test.tokens == []
def test_one_word():
under_test = build_tokenizer("Hello")
assert under_test.tokens == ["Hello"]
def test_one_word_with_spaces():
under_test = build_tokenizer("Hello ")
assert under_test.tokens == ["Hello"]
def test_two_words_with_spaces():
under_test = build_tokenizer("<p>lorem <span>jane-doe</span></p>")
assert under_test.tokens == ["lorem", "jane-doe"]
def test_words_inside_tags():
under_test = build_tokenizer("lorem<p>ipsum</p> dolor sit")
assert under_test.tokens == ["loremipsum", "dolor", "sit"]
def test_one_mention():
under_test = build_tokenizer_with_data(
"<span class=\"entity\" data-entity-id=\"0\">@jane-doe</span>",
"{\"0\":{\"id\":[{\"type\":\"com.symphony.user.userId\",\"value\":\"12345678\"}],"
"\"type\":\"com.symphony.user.mention\"}}")
assert under_test.tokens == [Mention("@jane-doe", 12345678)]
assert under_test.tokens[0].user_display_name == "jane-doe"
def test_mention_without_data():
under_test = build_tokenizer("<span class=\"entity\" data-entity-id=\"0\">@jane-doe</span>")
assert under_test.tokens == ["@jane-doe"], \
"The mentions is handled as a string argument since no json entity provided"
def test_two_mentions_with_space():
under_test = build_tokenizer_with_data(
"<div><span class=\"entity\" data-entity-id=\"0\">@jane-doe</span> "
"<span class=\"entity\" data-entity-id=\"1\">@John Doe</span></div>",
"{\"0\":{\"id\":[{\"type\":\"com.symphony.user.userId\",\"value\":\"12345678\"}],"
"\"type\":\"com.symphony.user.mention\"},\"1\":{\"id\":[{\"type\":\"com.symphony.user.userId\","
"\"value\":\"12345679\"}],\"type\":\"com.symphony.user.mention\"}}")
assert under_test.tokens == [Mention("@jane-doe", 12345678), Mention("@John Doe", 12345679)]
assert under_test.tokens[0].user_display_name == "jane-doe"
assert under_test.tokens[1].user_display_name == "John Doe"
def test_two_mentions_without_space():
under_test = build_tokenizer_with_data(
"<div><span class=\"entity\" data-entity-id=\"0\">@jane-doe</span>"
"<span class=\"entity\" data-entity-id=\"1\">@John Doe</span></div>",
"{\"0\":{\"id\":[{\"type\":\"com.symphony.user.userId\",\"value\":\"12345678\"}],"
"\"type\":\"com.symphony.user.mention\"},\"1\":{\"id\":[{\"type\":\"com.symphony.user.userId\","
"\"value\":\"12345679\"}],\"type\":\"com.symphony.user.mention\"}}")
assert under_test.tokens == [Mention("@jane-doe", 12345678), Mention("@John Doe", 12345679)]
assert under_test.tokens[0].user_display_name == "jane-doe"
assert under_test.tokens[1].user_display_name == "John Doe"
def test_text_and_one_mention():
under_test = build_tokenizer_with_data(
"<div>lorem<span class=\"entity\" data-entity-id=\"0\">@jane-doe</span></div>",
"{\"0\":{\"id\":[{\"type\":\"com.symphony.user.userId\",\"value\":\"12345678\"}],"
"\"type\":\"com.symphony.user.mention\"}}")
assert under_test.tokens == ["lorem", Mention("@jane-doe", 12345678)]
assert under_test.tokens[1].user_display_name == "jane-doe"
def test_text_and_two_mentions():
under_test = build_tokenizer_with_data(
"<div>lorem<span class=\"entity\" data-entity-id=\"0\">@jane-doe</span>"
"<span class=\"entity\" data-entity-id=\"1\">@jane-doe-two</span></div>",
"{\"0\":{\"id\":[{\"type\":\"com.symphony.user.userId\",\"value\":\"12345678\"}],"
"\"type\":\"com.symphony.user.mention\"},"
"\"1\":{\"id\":[{\"type\":\"com.symphony.user.userId\",\"value\":\"12345679\"}],"
"\"type\":\"com.symphony.user.mention\"}}")
assert under_test.tokens == ["lorem", Mention("@jane-doe", 12345678), Mention("@jane-doe-two", 12345679)]
assert under_test.tokens[1].user_display_name == "jane-doe"
assert under_test.tokens[2].user_display_name == "jane-doe-two"
def test_one_hashtag():
under_test = build_tokenizer_with_data(
"<div><span class=\"entity\" data-entity-id=\"0\">#myhashtag</span></div>",
"{\"0\":{\"id\":[{\"type\":\"org.symphonyoss.taxonomy.hashtag\",\"value\":\"hashtag\"}],"
"\"type\":\"org.symphonyoss.taxonomy\"}}")
assert under_test.tokens == [Hashtag("#myhashtag", "hashtag")]
def test_words_and_hashtag():
under_test = build_tokenizer_with_data(
"<div>mytext1<span class=\"entity\" data-entity-id=\"0\">#myhashtag</span> mytext2</div>",
"{\"0\":{\"id\":[{\"type\":\"org.symphonyoss.taxonomy.hashtag\",\"value\":\"hashtag\"}],"
"\"type\":\"org.symphonyoss.taxonomy\"}}")
assert under_test.tokens == ["mytext1", Hashtag("#myhashtag", "hashtag"), "mytext2"]
def test_words_and_hashtag_with_paragraph():
under_test = build_tokenizer_with_data(
"<div>mytext1<span class=\"entity\" data-entity-id=\"0\">#myhashtag</span> <p>mytext2</p></div>",
"{\"0\":{\"id\":[{\"type\":\"org.symphonyoss.taxonomy.hashtag\",\"value\":\"hashtag\"}],"
"\"type\":\"org.symphonyoss.taxonomy\"}}")
assert under_test.tokens == ["mytext1", Hashtag("#myhashtag", "hashtag"), "mytext2"]
def test_one_cashtag():
under_test = build_tokenizer_with_data(
"<span class=\"entity\" data-entity-id=\"0\">$mycashtag</span>",
"{\"0\":{\"id\":[{\"type\":\"org.symphonyoss.fin.security.id.ticker\",\"value\":\"mycashtag\"}],"
"\"type\":\"org.symphonyoss.fin.security\"}}")
assert under_test.tokens == [Cashtag("$mycashtag", "mycashtag")]
class PartialV4Message:
def __init__(self, message: str):
self.message = message
def test_partial_message():
tokenizer = InputTokenizer(PartialV4Message(message=build_v4_message("hello").message))
assert tokenizer.tokens == ["hello"]
def build_tokenizer(content):
return build_tokenizer_with_data(content, "{}")
def build_tokenizer_with_data(content, data):
return InputTokenizer(build_v4_message(content, data))
def build_v4_message(content, data="{}"):
return V4Message(attachments=[],
message="<div data-format=\"PresentationML\" data-version=\"2.0\" class=\"wysiwyg\"><p>" + content +
"</p></div>",
data=data)