From 8452299e8047c732fb0834e58a367216b99507d4 Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Sun, 26 Apr 2026 13:18:49 -0700 Subject: [PATCH 1/2] HDDS-10489. Use CRC tables to speed up galoisFieldMultiply in CrcUtil. --- .../ozone/common/PureJavaCrc32ByteBuffer.java | 12 ++ .../common/PureJavaCrc32CByteBuffer.java | 12 ++ .../ozone/client/checksum/CrcComposer.java | 49 +++--- .../hadoop/ozone/client/checksum/CrcUtil.java | 113 +++++++------- .../ozone/client/checksum/TestCrcUtil.java | 146 ++++++++++++++++++ 5 files changed, 247 insertions(+), 85 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/PureJavaCrc32ByteBuffer.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/PureJavaCrc32ByteBuffer.java index 7a14be17996b..23c363084702 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/PureJavaCrc32ByteBuffer.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/PureJavaCrc32ByteBuffer.java @@ -553,4 +553,16 @@ public final class PureJavaCrc32ByteBuffer extends ChecksumByteBuffer.CrcIntTabl int[] getTable() { return T; } + + /** + * Compute x mod p, where p is the CRC32 polynomial. + * @param x the input value + * @return x mod p + */ + public static int mod(long x) { + final int y = (int)(x); + return (int)(x >> 32) + ^ ((T[((y << 24) >>> 24) + 0x300] ^ T[((y << 16) >>> 24) + 0x200]) + ^ (T[((y << 8) >>> 24) + 0x100] ^ T[((y /* */) >>> 24) /* */])); + } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/PureJavaCrc32CByteBuffer.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/PureJavaCrc32CByteBuffer.java index 0be92ec7cf91..88a3b354e672 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/PureJavaCrc32CByteBuffer.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/PureJavaCrc32CByteBuffer.java @@ -557,4 +557,16 @@ public final class PureJavaCrc32CByteBuffer extends ChecksumByteBuffer.CrcIntTab int[] getTable() { return T; } + + /** + * Compute x mod p, where p is the CRC32C polynomial. + * @param x the input value + * @return x mod p + */ + public static int mod(long x) { + final int y = (int)(x); + return (int)(x >> 32) + ^ ((T[((y << 24) >>> 24) + 0x300] ^ T[((y << 16) >>> 24) + 0x200]) + ^ (T[((y << 8) >>> 24) + 0x100] ^ T[((y /* */) >>> 24) /* */])); + } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/CrcComposer.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/CrcComposer.java index ca00c61558b2..25b567873db1 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/CrcComposer.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/CrcComposer.java @@ -20,8 +20,11 @@ import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.IOException; +import java.util.function.ToIntFunction; import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.annotation.InterfaceStability; +import org.apache.hadoop.ozone.common.PureJavaCrc32ByteBuffer; +import org.apache.hadoop.ozone.common.PureJavaCrc32CByteBuffer; import org.apache.hadoop.util.DataChecksum; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,11 +37,11 @@ */ @InterfaceAudience.Private @InterfaceStability.Unstable -public class CrcComposer { +public final class CrcComposer { private static final int CRC_SIZE_BYTES = 4; private static final Logger LOG = LoggerFactory.getLogger(CrcComposer.class); - private final int crcPolynomial; + private final ToIntFunction mod; private final int precomputedMonomialForHint; private final long bytesPerCrcHint; private final long stripeLength; @@ -76,28 +79,26 @@ public static CrcComposer newCrcComposer(DataChecksum.Type type, long bytesPerCr * underlying data size which aligns with the specified stripe boundary. */ public static CrcComposer newStripedCrcComposer(DataChecksum.Type type, long bytesPerCrcHint, long stripeLength) { - final int polynomial = CrcUtil.getCrcPolynomialForType(type); - return new CrcComposer( - polynomial, - CrcUtil.getMonomial(bytesPerCrcHint, polynomial), - bytesPerCrcHint, - stripeLength); + return new CrcComposer(type, bytesPerCrcHint, stripeLength); } - CrcComposer( - int crcPolynomial, - int precomputedMonomialForHint, - long bytesPerCrcHint, - long stripeLength) { - LOG.debug( - "crcPolynomial=0x{}, precomputedMonomialForHint=0x{}, " - + "bytesPerCrcHint={}, stripeLength={}", - Integer.toString(crcPolynomial, 16), - Integer.toString(precomputedMonomialForHint, 16), - bytesPerCrcHint, - stripeLength); - this.crcPolynomial = crcPolynomial; - this.precomputedMonomialForHint = precomputedMonomialForHint; + /** @return the mod function for the given type. */ + static ToIntFunction getModFunction(DataChecksum.Type type) { + switch (type) { + case CRC32: + return PureJavaCrc32ByteBuffer::mod; + case CRC32C: + return PureJavaCrc32CByteBuffer::mod; + default: + throw new IllegalArgumentException("Unexpected type: " + type); + } + } + + private CrcComposer(DataChecksum.Type type, long bytesPerCrcHint, long stripeLength) { + LOG.debug("type={}, bytesPerCrcHint={}, stripeLength={}", + type, bytesPerCrcHint, stripeLength); + this.mod = getModFunction(type); + this.precomputedMonomialForHint = CrcUtil.getMonomial(bytesPerCrcHint, mod); this.bytesPerCrcHint = bytesPerCrcHint; this.stripeLength = stripeLength; } @@ -157,9 +158,9 @@ public void update(int crcB, long bytesPerCrc) { if (curCompositeCrc == 0) { curCompositeCrc = crcB; } else if (bytesPerCrc == bytesPerCrcHint) { - curCompositeCrc = CrcUtil.composeWithMonomial(curCompositeCrc, crcB, precomputedMonomialForHint, crcPolynomial); + curCompositeCrc = CrcUtil.composeWithMonomial(curCompositeCrc, crcB, precomputedMonomialForHint, mod); } else { - curCompositeCrc = CrcUtil.compose(curCompositeCrc, crcB, bytesPerCrc, crcPolynomial); + curCompositeCrc = CrcUtil.compose(curCompositeCrc, crcB, bytesPerCrc, mod); } curPositionInStripe += bytesPerCrc; diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/CrcUtil.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/CrcUtil.java index 96eaeabeffff..ddc1bb0005b1 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/CrcUtil.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/CrcUtil.java @@ -18,9 +18,9 @@ package org.apache.hadoop.ozone.client.checksum; import java.util.Arrays; +import java.util.function.ToIntFunction; import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.annotation.InterfaceStability; -import org.apache.hadoop.util.DataChecksum; /** * This class provides utilities for working with CRCs. @@ -29,8 +29,7 @@ @InterfaceStability.Unstable public final class CrcUtil { public static final int MULTIPLICATIVE_IDENTITY = 0x80000000; - public static final int GZIP_POLYNOMIAL = 0xEDB88320; - public static final int CASTAGNOLI_POLYNOMIAL = 0x82F63B78; + private static final long UNIT = 0x8000_0000_0000_0000L; /** * Hide default constructor for a static utils class. @@ -39,22 +38,51 @@ private CrcUtil() { } /** - * getCrcPolynomialForType. - * - * @param type type. - * @return the int representation of the polynomial associated with the - * CRC {@code type}, suitable for use with further CRC arithmetic. + * @return a * b (mod p), + * where mod p is computed by the given mod function. */ - public static int getCrcPolynomialForType(DataChecksum.Type type) { - switch (type) { - case CRC32: - return GZIP_POLYNOMIAL; - case CRC32C: - return CASTAGNOLI_POLYNOMIAL; - default: - throw new IllegalArgumentException( - "No CRC polynomial could be associated with type: " + type); - } + static int multiplyMod(int a, int b, ToIntFunction mod) { + final long left = ((long)a) << 32; + final long right = ((long)b) << 32; + + final long product + = ((((((left & (UNIT /* */)) == 0L ? 0L : right) + ^ ((left & (UNIT >>> 1)) == 0L ? 0L : right >>> 1)) + ^ (((left & (UNIT >>> 2)) == 0L ? 0L : right >>> 2) + ^ ((left & (UNIT >>> 3)) == 0L ? 0L : right >>> 3))) + ^ ((((left & (UNIT >>> 4)) == 0L ? 0L : right >>> 4) + ^ ((left & (UNIT >>> 5)) == 0L ? 0L : right >>> 5)) + ^ (((left & (UNIT >>> 6)) == 0L ? 0L : right >>> 6) + ^ ((left & (UNIT >>> 7)) == 0L ? 0L : right >>> 7)))) + + ^ (((((left & (UNIT >>> 8)) == 0L ? 0L : right >>> 8) + ^ ((left & (UNIT >>> 9)) == 0L ? 0L : right >>> 9)) + ^ (((left & (UNIT >>> 10)) == 0L ? 0L : right >>> 10) + ^ ((left & (UNIT >>> 11)) == 0L ? 0L : right >>> 11))) + ^ ((((left & (UNIT >>> 12)) == 0L ? 0L : right >>> 12) + ^ ((left & (UNIT >>> 13)) == 0L ? 0L : right >>> 13)) + ^ (((left & (UNIT >>> 14)) == 0L ? 0L : right >>> 14) + ^ ((left & (UNIT >>> 15)) == 0L ? 0L : right >>> 15))))) + + ^ ((((((left & (UNIT >>> 16)) == 0L ? 0L : right >>> 16) + ^ ((left & (UNIT >>> 17)) == 0L ? 0L : right >>> 17)) + ^ (((left & (UNIT >>> 18)) == 0L ? 0L : right >>> 18) + ^ ((left & (UNIT >>> 19)) == 0L ? 0L : right >>> 19))) + ^ ((((left & (UNIT >>> 20)) == 0L ? 0L : right >>> 20) + ^ ((left & (UNIT >>> 21)) == 0L ? 0L : right >>> 21)) + ^ (((left & (UNIT >>> 22)) == 0L ? 0L : right >>> 22) + ^ ((left & (UNIT >>> 23)) == 0L ? 0L : right >>> 23)))) + + ^ (((((left & (UNIT >>> 24)) == 0L ? 0L : right >>> 24) + ^ ((left & (UNIT >>> 25)) == 0L ? 0L : right >>> 25)) + ^ (((left & (UNIT >>> 26)) == 0L ? 0L : right >>> 26) + ^ ((left & (UNIT >>> 27)) == 0L ? 0L : right >>> 27))) + ^ ((((left & (UNIT >>> 28)) == 0L ? 0L : right >>> 28) + ^ ((left & (UNIT >>> 29)) == 0L ? 0L : right >>> 29)) + ^ (((left & (UNIT >>> 30)) == 0L ? 0L : right >>> 30) + ^ ((left & (UNIT >>> 31)) == 0L ? 0L : right >>> 31))))); + + return mod.applyAsInt(product); } /** @@ -66,7 +94,7 @@ public static int getCrcPolynomialForType(DataChecksum.Type type) { * @param mod mod. * @return monomial. */ - public static int getMonomial(long lengthBytes, int mod) { + public static int getMonomial(long lengthBytes, ToIntFunction mod) { if (lengthBytes == 0) { return MULTIPLICATIVE_IDENTITY; } else if (lengthBytes < 0) { @@ -85,9 +113,9 @@ public static int getMonomial(long lengthBytes, int mod) { while (degree > 0) { if ((degree & 1) != 0) { product = (product == MULTIPLICATIVE_IDENTITY) ? multiplier : - galoisFieldMultiply(product, multiplier, mod); + multiplyMod(product, multiplier, mod); } - multiplier = galoisFieldMultiply(multiplier, multiplier, mod); + multiplier = multiplyMod(multiplier, multiplier, mod); degree >>= 1; } return product; @@ -103,8 +131,8 @@ public static int getMonomial(long lengthBytes, int mod) { * @return compose with monomial. */ public static int composeWithMonomial( - int crcA, int crcB, int monomial, int mod) { - return galoisFieldMultiply(crcA, monomial, mod) ^ crcB; + int crcA, int crcB, int monomial, ToIntFunction mod) { + return multiplyMod(crcA, monomial, mod) ^ crcB; } /** @@ -116,7 +144,7 @@ public static int composeWithMonomial( * @param mod mod. * @return compose result. */ - public static int compose(int crcA, int crcB, long lengthB, int mod) { + public static int compose(int crcA, int crcB, long lengthB, ToIntFunction mod) { int monomial = getMonomial(lengthB, mod); return composeWithMonomial(crcA, crcB, monomial, mod); } @@ -216,41 +244,4 @@ public static String toMultiCrcString(final byte[] bytes) { sb.append(']'); return sb.toString(); } - - /** - * Galois field multiplication of {@code p} and {@code q} with the - * generator polynomial {@code m} as the modulus. - * - * @param m The little-endian polynomial to use as the modulus when - * multiplying p and q, with implicit "1" bit beyond the bottom bit. - */ - private static int galoisFieldMultiply(int p, int q, int m) { - int summation = 0; - - // Top bit is the x^0 place; each right-shift increments the degree of the - // current term. - int curTerm = MULTIPLICATIVE_IDENTITY; - - // Iteratively multiply p by x mod m as we go to represent the q[i] term - // (of degree x^i) times p. - int px = p; - - while (curTerm != 0) { - if ((q & curTerm) != 0) { - summation ^= px; - } - - // Bottom bit represents highest degree since we're little-endian; before - // we multiply by "x" for the next term, check bottom bit to know whether - // the resulting px will thus have a term matching the implicit "1" term - // of "m" and thus will need to subtract "m" after mutiplying by "x". - boolean hasMaxDegree = ((px & 1) != 0); - px >>>= 1; - if (hasMaxDegree) { - px ^= m; - } - curTerm >>>= 1; - } - return summation; - } } diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/client/checksum/TestCrcUtil.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/client/checksum/TestCrcUtil.java index 3b4cc2e9016c..50c9d57ba493 100644 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/client/checksum/TestCrcUtil.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/client/checksum/TestCrcUtil.java @@ -21,12 +21,17 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.Random; +import java.util.function.ToIntFunction; +import org.apache.hadoop.util.DataChecksum; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; /** Test {@link CrcUtil}. */ @Timeout(10) public class TestCrcUtil { + private static final Random RANDOM = new Random(); + /** Assert that the given throwable contains the given message. */ public static void assertContains(Throwable t, String message) { assertTrue(t.getMessage().contains(message), () -> "Message \"" + message + "\" not found: " + t); @@ -87,4 +92,145 @@ public void testToMultiCrcStringSingleElement() { public void testToMultiCrcStringNoElements() { assertEquals("[]", CrcUtil.toMultiCrcString(new byte[0])); } + + @Test + public void testMultiplyMod() { + runTestMultiplyMod(10_000_000, DataChecksum.Type.CRC32); + runTestMultiplyMod(10_000_000, DataChecksum.Type.CRC32C); + } + + private static long[] runTestMultiplyMod(int n, DataChecksum.Type type) { + System.out.printf("Run %s with %d computations%n", type, n); + final int polynomial = getCrcPolynomialForType(type); + final ToIntFunction mod = CrcComposer.getModFunction(type); + + final int[] p = new int[n]; + final int[] q = new int[n]; + for (int i = 0; i < n; i++) { + p[i] = RANDOM.nextInt(); + q[i] = RANDOM.nextInt(); + } + + final int[] expected = new int[n]; + final long[] times = new long[2]; + final long t0 = System.currentTimeMillis(); + for (int i = 0; i < n; i++) { + expected[i] = galoisFieldMultiply(p[i], q[i], polynomial); + } + times[0] = System.currentTimeMillis() - t0; + final double ops0 = n * 1000.0 / times[0]; + System.out.printf("galoisFieldMultiply: %.3fs (%.2f ops)%n", times[0] / 1000.0, ops0); + + final int[] computed = new int[n]; + final long t1 = System.currentTimeMillis(); + for (int i = 0; i < n; i++) { + computed[i] = CrcUtil.multiplyMod(p[i], q[i], mod); + } + times[1] = System.currentTimeMillis() - t1; + final double ops1 = n * 1000.0 / times[1]; + System.out.printf("tableMultiply : %.3fs (%.2f ops)%n", times[1] / 1000.0, ops1); + System.out.printf("tableMultiply is %.2f%% faster%n", (ops1 - ops0) * 100.0 / ops0); + + for (int i = 0; i < n; i++) { + if (expected[i] != computed[i]) { + System.out.printf("expected %08X%n", expected[i]); + System.out.printf("computed %08X%n", computed[i]); + throw new IllegalStateException(); + } + } + return times; + } + + /** + * getCrcPolynomialForType. + * + * @param type type. + * @return the int representation of the polynomial associated with the + * CRC {@code type}, suitable for use with further CRC arithmetic. + */ + private static int getCrcPolynomialForType(DataChecksum.Type type) { + switch (type) { + case CRC32: + return org.apache.hadoop.util.CrcUtil.GZIP_POLYNOMIAL; + case CRC32C: + return org.apache.hadoop.util.CrcUtil.CASTAGNOLI_POLYNOMIAL; + default: + throw new IllegalArgumentException("Unexpected type: " + type); + } + } + + /** + * Galois field multiplication of {@code p} and {@code q} with the + * generator polynomial {@code m} as the modulus. + * + * @param m The little-endian polynomial to use as the modulus when + * multiplying p and q, with implicit "1" bit beyond the bottom bit. + */ + private static int galoisFieldMultiply(int p, int q, int m) { + int summation = 0; + + // Top bit is the x^0 place; each right-shift increments the degree of the + // current term. + int curTerm = org.apache.hadoop.util.CrcUtil.MULTIPLICATIVE_IDENTITY; + + // Iteratively multiply p by x mod m as we go to represent the q[i] term + // (of degree x^i) times p. + int px = p; + + while (curTerm != 0) { + if ((q & curTerm) != 0) { + summation ^= px; + } + + // Bottom bit represents highest degree since we're little-endian; before + // we multiply by "x" for the next term, check bottom bit to know whether + // the resulting px will thus have a term matching the implicit "1" term + // of "m" and thus will need to subtract "m" after mutiplying by "x". + boolean hasMaxDegree = ((px & 1) != 0); + px >>>= 1; + if (hasMaxDegree) { + px ^= m; + } + curTerm >>>= 1; + } + return summation; + } + + /** For running benchmarks. */ + public static class Benchmark { + /** + * Usages: java {@link Benchmark} [m] [n] [type] + * m: the number of iterations + * n: the number of multiplication + * type: the CRC type, either CRC32 or CRC32C. + */ + public static void main(String[] args) { + final int m = args.length >= 1 ? Integer.parseInt(args[0]) : 10; + final int n = args.length >= 2 ? Integer.parseInt(args[1]) : 100_000_000; + final DataChecksum.Type type = args.length >= 3 ? DataChecksum.Type.valueOf(args[2]) + : DataChecksum.Type.CRC32; + + final int warmUpIterations = 2; + System.out.printf("%nStart warming up with %d iterations ...%n", warmUpIterations); + for (int i = 0; i < 2; i++) { + runTestMultiplyMod(n, type); + } + + System.out.printf("%nStart benchmark with %d iterations ...%n", m); + final long[] times = new long[2]; + for (int i = 0; i < m; i++) { + System.out.printf("%d) ", i); + final long[] t = runTestMultiplyMod(n, type); + times[0] += t[0]; + times[1] += t[1]; + } + + System.out.printf("%nResult) %d x %d computations:%n", m, n); + final double ops0 = m * n * 1000.0 / times[0]; + System.out.printf("galoisFieldMultiply: %.3fs (%.2f ops)%n", times[0] / 1000.0, ops0); + final double ops1 = m * n * 1000.0 / times[1]; + System.out.printf("tableMultiply : %.3fs (%.2f ops)%n", times[1] / 1000.0, ops1); + System.out.printf("tableMultiply is %.2f%% faster%n", (ops1 - ops0) * 100.0 / ops0); + } + } } From ed7938200432af91687f569e405c97117f48ddf6 Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Wed, 29 Apr 2026 09:42:21 -0700 Subject: [PATCH 2/2] Use LongToIntFunction --- .../hadoop/ozone/client/checksum/CrcComposer.java | 6 +++--- .../apache/hadoop/ozone/client/checksum/CrcUtil.java | 10 +++++----- .../hadoop/ozone/client/checksum/TestCrcUtil.java | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/CrcComposer.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/CrcComposer.java index 25b567873db1..03cb90bf99f3 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/CrcComposer.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/CrcComposer.java @@ -20,7 +20,7 @@ import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.IOException; -import java.util.function.ToIntFunction; +import java.util.function.LongToIntFunction; import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.annotation.InterfaceStability; import org.apache.hadoop.ozone.common.PureJavaCrc32ByteBuffer; @@ -41,7 +41,7 @@ public final class CrcComposer { private static final int CRC_SIZE_BYTES = 4; private static final Logger LOG = LoggerFactory.getLogger(CrcComposer.class); - private final ToIntFunction mod; + private final LongToIntFunction mod; private final int precomputedMonomialForHint; private final long bytesPerCrcHint; private final long stripeLength; @@ -83,7 +83,7 @@ public static CrcComposer newStripedCrcComposer(DataChecksum.Type type, long byt } /** @return the mod function for the given type. */ - static ToIntFunction getModFunction(DataChecksum.Type type) { + static LongToIntFunction getModFunction(DataChecksum.Type type) { switch (type) { case CRC32: return PureJavaCrc32ByteBuffer::mod; diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/CrcUtil.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/CrcUtil.java index ddc1bb0005b1..6cf414dfbd43 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/CrcUtil.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/CrcUtil.java @@ -18,7 +18,7 @@ package org.apache.hadoop.ozone.client.checksum; import java.util.Arrays; -import java.util.function.ToIntFunction; +import java.util.function.LongToIntFunction; import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.annotation.InterfaceStability; @@ -41,7 +41,7 @@ private CrcUtil() { * @return a * b (mod p), * where mod p is computed by the given mod function. */ - static int multiplyMod(int a, int b, ToIntFunction mod) { + static int multiplyMod(int a, int b, LongToIntFunction mod) { final long left = ((long)a) << 32; final long right = ((long)b) << 32; @@ -94,7 +94,7 @@ static int multiplyMod(int a, int b, ToIntFunction mod) { * @param mod mod. * @return monomial. */ - public static int getMonomial(long lengthBytes, ToIntFunction mod) { + public static int getMonomial(long lengthBytes, LongToIntFunction mod) { if (lengthBytes == 0) { return MULTIPLICATIVE_IDENTITY; } else if (lengthBytes < 0) { @@ -131,7 +131,7 @@ public static int getMonomial(long lengthBytes, ToIntFunction mod) { * @return compose with monomial. */ public static int composeWithMonomial( - int crcA, int crcB, int monomial, ToIntFunction mod) { + int crcA, int crcB, int monomial, LongToIntFunction mod) { return multiplyMod(crcA, monomial, mod) ^ crcB; } @@ -144,7 +144,7 @@ public static int composeWithMonomial( * @param mod mod. * @return compose result. */ - public static int compose(int crcA, int crcB, long lengthB, ToIntFunction mod) { + public static int compose(int crcA, int crcB, long lengthB, LongToIntFunction mod) { int monomial = getMonomial(lengthB, mod); return composeWithMonomial(crcA, crcB, monomial, mod); } diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/client/checksum/TestCrcUtil.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/client/checksum/TestCrcUtil.java index 50c9d57ba493..e979d0f2bdd2 100644 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/client/checksum/TestCrcUtil.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/client/checksum/TestCrcUtil.java @@ -22,7 +22,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Random; -import java.util.function.ToIntFunction; +import java.util.function.LongToIntFunction; import org.apache.hadoop.util.DataChecksum; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; @@ -102,7 +102,7 @@ public void testMultiplyMod() { private static long[] runTestMultiplyMod(int n, DataChecksum.Type type) { System.out.printf("Run %s with %d computations%n", type, n); final int polynomial = getCrcPolynomialForType(type); - final ToIntFunction mod = CrcComposer.getModFunction(type); + final LongToIntFunction mod = CrcComposer.getModFunction(type); final int[] p = new int[n]; final int[] q = new int[n];