Skip to content

Commit efd0a78

Browse files
oschwaldoschwald
authored
Merge pull request #157 from maxmind/horgh/email-normalize
Normalize email before hashing
2 parents 0fe7914 + db3b8d0 commit efd0a78

4 files changed

Lines changed: 159 additions & 3 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ CHANGELOG
77
* The following payment processors were added to the `Payment.Processor` enum:
88
* `APPLE_PAY`
99
* `APS_PAYMENTS`
10+
* Added additional normalizing of the email address if `hashAddress` is
11+
enabled.
1012

1113
1.16.0 (2020-10-14)
1214
-------------------

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -285,6 +285,6 @@ This API uses [Semantic Versioning](https://semver.org/).
285285

286286
## Copyright and License ##
287287

288-
This software is Copyright (c) 2015-2020 by MaxMind, Inc.
288+
This software is Copyright (c) 2015-2021 by MaxMind, Inc.
289289

290290
This is free software, licensed under the Apache License, Version 2.0.

‎src/main/java/com/maxmind/minfraud/request/Email.java‎

Lines changed: 71 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@
66
import org.apache.commons.codec.digest.DigestUtils;
77
import org.apache.commons.validator.routines.DomainValidator;
88
import org.apache.commons.validator.routines.EmailValidator;
9+
import java.net.IDN;
10+
import java.util.Collections;
11+
import java.util.HashMap;
12+
import java.util.Map;
913

1014
/**
1115
* The email information for the transaction.
@@ -14,6 +18,24 @@ public final class Email extends AbstractModel {
1418
private final String address;
1519
private final boolean hashAddress;
1620
private final String domain;
21+
private static final Map<String, String> typoDomains;
22+
23+
static {
24+
HashMap<String, String> m = new HashMap<String, String>() {{
25+
// gmail.com
26+
put("35gmai.com", "gmail.com");
27+
put("636gmail.com", "gmail.com");
28+
put("gamil.com", "gmail.com");
29+
put("gmail.comu", "gmail.com");
30+
put("gmial.com", "gmail.com");
31+
put("gmil.com", "gmail.com");
32+
put("yahoogmail.com", "gmail.com");
33+
// outlook.com
34+
put("putlook.com", "outlook.com");
35+
}};
36+
37+
typoDomains = Collections.unmodifiableMap(m);
38+
}
1739

1840
private Email(Email.Builder builder) {
1941
address = builder.address;
@@ -128,11 +150,58 @@ public String getAddress() {
128150
return null;
129151
}
130152
if (hashAddress) {
131-
return DigestUtils.md5Hex(address.toLowerCase());
153+
return DigestUtils.md5Hex(cleanAddress(address));
132154
}
133155
return address;
134156
}
135157

158+
private String cleanAddress(String address) {
159+
address = address.trim().toLowerCase();
160+
161+
int domainIndex = address.lastIndexOf('@');
162+
if (domainIndex == -1 || domainIndex + 1 == address.length()) {
163+
return address;
164+
}
165+
166+
String localPart = address.substring(0, domainIndex);
167+
String domain = address.substring(domainIndex + 1);
168+
169+
domain = cleanDomain(domain);
170+
171+
int stopChar;
172+
if (domain.equals("yahoo.com")) {
173+
stopChar = '-';
174+
} else {
175+
stopChar = '+';
176+
}
177+
int stopCharIndex = localPart.indexOf(stopChar);
178+
if (stopCharIndex > 0) {
179+
localPart = localPart.substring(0, stopCharIndex);
180+
}
181+
182+
return localPart + "@" + domain;
183+
}
184+
185+
private String cleanDomain(String domain) {
186+
if (domain == null) {
187+
return null;
188+
}
189+
190+
domain = domain.trim();
191+
192+
if (domain.endsWith(".")) {
193+
domain = domain.substring(0, domain.length() - 1);
194+
}
195+
196+
domain = IDN.toASCII(domain);
197+
198+
if (typoDomains.containsKey(domain)) {
199+
domain = typoDomains.get(domain);
200+
}
201+
202+
return domain;
203+
}
204+
136205
/**
137206
* @return The MD5 hash of the email address if you set an address using
138207
* {@link Builder#address(String)}, or null if you did not.
@@ -144,7 +213,7 @@ public String getAddressMd5() {
144213
if (address == null) {
145214
return null;
146215
}
147-
return DigestUtils.md5Hex(address.toLowerCase());
216+
return DigestUtils.md5Hex(cleanAddress(address));
148217
}
149218

150219
/**

‎src/test/java/com/maxmind/minfraud/request/EmailTest.java‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,91 @@ public void testGetAddressWithoutSettingIt() throws Exception {
8383
assertNull("null addressMd5 if none set", email2.getAddressMd5());
8484
}
8585

86+
@Test
87+
public void testNormalizing() throws Exception {
88+
Email e;
89+
90+
e = new Builder().address("test@maxmind.com").hashAddress().build();
91+
assertEquals("MD5", "977577b140bfb7c516e4746204fbdb01", e.getAddress());
92+
assertEquals("domain", "maxmind.com", e.getDomain());
93+
94+
e = new Builder().address("Test@maxmind.com").hashAddress().build();
95+
assertEquals("MD5", "977577b140bfb7c516e4746204fbdb01", e.getAddress());
96+
assertEquals("domain", "maxmind.com", e.getDomain());
97+
98+
e = new Builder(false).address(" Test@maxmind.com").hashAddress().build();
99+
assertEquals("MD5", "977577b140bfb7c516e4746204fbdb01", e.getAddress());
100+
assertEquals("domain", "maxmind.com", e.getDomain());
101+
102+
e = new Builder().address("Test+alias@maxmind.com").hashAddress().build();
103+
assertEquals("MD5", "977577b140bfb7c516e4746204fbdb01", e.getAddress());
104+
assertEquals("domain", "maxmind.com", e.getDomain());
105+
106+
e = new Builder().address("Test+007+008@maxmind.com").hashAddress().build();
107+
assertEquals("MD5", "977577b140bfb7c516e4746204fbdb01", e.getAddress());
108+
assertEquals("domain", "maxmind.com", e.getDomain());
109+
110+
e = new Builder().address("Test+@maxmind.com").hashAddress().build();
111+
assertEquals("MD5", "977577b140bfb7c516e4746204fbdb01", e.getAddress());
112+
assertEquals("domain", "maxmind.com", e.getDomain());
113+
114+
e = new Builder(false).address("Test@maxmind.com.").hashAddress().build();
115+
assertEquals("MD5", "977577b140bfb7c516e4746204fbdb01", e.getAddress());
116+
assertEquals("domain", "maxmind.com.", e.getDomain());
117+
118+
e = new Builder().address("+@maxmind.com").hashAddress().build();
119+
assertEquals("MD5", "aa57884e48f0dda9fc6f4cb2bffb1dd2", e.getAddress());
120+
assertEquals("domain", "maxmind.com", e.getDomain());
121+
122+
e = new Builder(false).address("Test@ maxmind.com").hashAddress().build();
123+
assertEquals("MD5", "977577b140bfb7c516e4746204fbdb01", e.getAddress());
124+
assertEquals("domain", " maxmind.com", e.getDomain());
125+
126+
e = new Builder().address("Test+foo@yahoo.com").hashAddress().build();
127+
assertEquals("MD5", "a5f830c699fd71ad653aa59fa688c6d9", e.getAddress());
128+
assertEquals("domain", "yahoo.com", e.getDomain());
129+
130+
e = new Builder().address("Test-foo@yahoo.com").hashAddress().build();
131+
assertEquals("MD5", "88e478531ab3bc303f1b5da82c2e9bbb", e.getAddress());
132+
assertEquals("domain", "yahoo.com", e.getDomain());
133+
134+
e = new Builder().address("Test-foo-foo2@yahoo.com").hashAddress().build();
135+
assertEquals("MD5", "88e478531ab3bc303f1b5da82c2e9bbb", e.getAddress());
136+
assertEquals("domain", "yahoo.com", e.getDomain());
137+
138+
e = new Builder().address("Test-foo@gmail.com").hashAddress().build();
139+
assertEquals("MD5", "6f3ff986fa5e830dbbf08a942777a17c", e.getAddress());
140+
assertEquals("domain", "gmail.com", e.getDomain());
141+
142+
e = new Builder().address("test@gmail.com").hashAddress().build();
143+
assertEquals("MD5", "1aedb8d9dc4751e229a335e371db8058", e.getAddress());
144+
assertEquals("domain", "gmail.com", e.getDomain());
145+
146+
e = new Builder().address("test@gamil.com").hashAddress().build();
147+
assertEquals("MD5", "1aedb8d9dc4751e229a335e371db8058", e.getAddress());
148+
assertEquals("domain", "gamil.com", e.getDomain());
149+
150+
e = new Builder().address("test@bücher.com").hashAddress().build();
151+
assertEquals("MD5", "24948acabac551360cd510d5e5e2b464", e.getAddress());
152+
assertEquals("domain", "bücher.com", e.getDomain());
153+
154+
e = new Builder().address("Test+alias@Bücher.com").hashAddress().build();
155+
assertEquals("MD5", "24948acabac551360cd510d5e5e2b464", e.getAddress());
156+
assertEquals("domain", "Bücher.com", e.getDomain());
157+
158+
e = new Builder(false).address("test").hashAddress().build();
159+
assertEquals("MD5", "098f6bcd4621d373cade4e832627b4f6", e.getAddress());
160+
assertNull("domain", e.getDomain());
161+
162+
e = new Builder(false).address("test@").hashAddress().build();
163+
assertEquals("MD5", "246a848af2f8394e3adbc738dbe43720", e.getAddress());
164+
assertNull("domain", e.getDomain());
165+
166+
e = new Builder(false).address("test@.").hashAddress().build();
167+
assertEquals("MD5", "246a848af2f8394e3adbc738dbe43720", e.getAddress());
168+
assertEquals("domain", ".", e.getDomain());
169+
}
170+
86171
@Test(expected = IllegalArgumentException.class)
87172
public void testInvalidAddress() throws Exception {
88173
new Builder().address("a@test@test.org").build();

0 commit comments

Comments
 (0)