diff --git a/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/CIDR.java b/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/CIDR.java index 3add082a81..1d97e67280 100644 --- a/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/CIDR.java +++ b/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/CIDR.java @@ -27,10 +27,16 @@ * defined by the CIDR. */ public class CIDR { - InetAddress addr; - int mask; - public CIDR(InetAddress address, int mask) { + private final InetAddress addr; + private final int mask; + + public CIDR(InetAddress address, int mask) throws IllegalArgumentException { + int maxMask = address.getAddress().length * 8; + if (mask < 0 || mask > maxMask) { + throw new IllegalArgumentException( + "Invalid CIDR mask /" + mask + " for " + address); + } this.addr = address; this.mask = mask; } @@ -42,16 +48,23 @@ public CIDR(String cidr) throws IllegalArgumentException { ipStr = cidr.substring(0, sep); } addr = InetAddresses.forString(ipStr); + int parsedMask; if (sep > -1) { - mask = Integer.parseInt(cidr.substring(sep + 1)); + parsedMask = Integer.parseInt(cidr.substring(sep + 1)); } else { - mask = addr.getAddress().length * 8; + parsedMask = addr.getAddress().length * 8; } if (cidr.indexOf(':') > -1 && addr.getAddress().length == 4) { // IPv4-mapped IPv6 addresses are automatically converted to IPv4, // need to shift the mask - mask = Math.max(0, mask - 96); + parsedMask = Math.max(0, parsedMask - 96); + } + int maxMask = addr.getAddress().length * 8; + if (parsedMask < 0 || parsedMask > maxMask) { + throw new IllegalArgumentException( + "Invalid CIDR mask /" + parsedMask + " for " + ipStr); } + this.mask = parsedMask; } public boolean contains(InetAddress address) { @@ -63,11 +76,18 @@ public boolean contains(InetAddress address) { } for (int i = 0; i < addr0.length; i++) { int remainingMaskBits = mask - (i * 8); - if (remainingMaskBits <= 0) + if (remainingMaskBits <= 0) { return true; - int m = ~(0xff >> remainingMaskBits); // mask for byte under cursor - if ((addr0[i] & m) != (addr1[i] & m)) + } + /* + * keep the mask within one byte so the shift does not wrap (Java shifts + * mod 32) + */ + int m = remainingMaskBits >= 8 ? 0xff + : (0xff << (8 - remainingMaskBits)) & 0xff; + if ((addr0[i] & m) != (addr1[i] & m)) { return false; + } } return true; } diff --git a/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/IPFilterRules.java b/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/IPFilterRules.java index a9252ddc5a..5ba0a9fd64 100644 --- a/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/IPFilterRules.java +++ b/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/IPFilterRules.java @@ -105,15 +105,15 @@ private static List> parseIPRules(Configuration conf, switch (ipRule.toLowerCase(Locale.ROOT)) { case "localhost": case "loopback": - rules.add((InetAddress a) -> a.isLoopbackAddress()); + rules.add(InetAddress::isLoopbackAddress); break; case "sitelocal": - rules.add((InetAddress a) -> a.isSiteLocalAddress()); + rules.add(InetAddress::isSiteLocalAddress); break; default: try { CIDR cidr = new CIDR(ipRule); - rules.add((InetAddress a) -> cidr.contains(a)); + rules.add(cidr::contains); } catch (IllegalArgumentException e) { LOG.error( "Failed to parse {} as CIDR, ignoring to configure IP rules ({})", diff --git a/src/plugin/protocol-okhttp/src/test/org/apache/nutch/protocol/okhttp/TestIPAddressFiltering.java b/src/plugin/protocol-okhttp/src/test/org/apache/nutch/protocol/okhttp/TestIPAddressFiltering.java index 309c977b1c..63627bc976 100644 --- a/src/plugin/protocol-okhttp/src/test/org/apache/nutch/protocol/okhttp/TestIPAddressFiltering.java +++ b/src/plugin/protocol-okhttp/src/test/org/apache/nutch/protocol/okhttp/TestIPAddressFiltering.java @@ -28,6 +28,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -63,6 +64,17 @@ public void testCIDRnotContains(String cidr, String ip) { assertFalse(c.contains(i), i + " should not be in " + c); } + @Test + public void testInvalidCIDR() { + assertThrows(IllegalArgumentException.class, + () -> new CIDR(InetAddress.getByName("1.2.3.4"), 33)); + assertThrows(IllegalArgumentException.class, () -> new CIDR("1.2.3.4/33")); + assertThrows(IllegalArgumentException.class, () -> new CIDR("::ffff:7f00:0/129")); + assertThrows(IllegalArgumentException.class, () -> new CIDR("1.2.3.4/-1")); + // invalid rule + assertThrows(IllegalArgumentException.class, () -> new CIDR("foobar")); + } + /** Tests for {@link CIDR} */ @Test public void testCIDRs() { @@ -89,6 +101,13 @@ public void testCIDRs() { testCIDRnotContains("10.0.0.0/8", "9.255.255.255"); testCIDRnotContains("172.16.0.0/12", "172.32.0.0"); testCIDRnotContains("172.16.0.0/12", "171.255.255.255"); + + testCIDRnotContains("127.0.0.1/31", "0.0.0.1"); + // NUTCH-3187 CIDR edge cases - /32, /128 - test bit shifting + testCIDRnotContains("127.0.0.1", "0.0.0.1"); + testCIDRnotContains("127.0.0.1/32", "0.0.0.1"); + testCIDRnotContains("127.0.0.1", "255.0.0.1"); + testCIDRnotContains("fe80::2f29:b6f0:a4c:32ae/128", "::2f29:b6f0:a4c:32ae"); } public void testFilter(Configuration conf, String[] included, String[] excluded) {