Enforce UTF8 when decoding byte[] to string in ValueReader (#16608)

Author: jtao15

pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/ValueReader.java

@@ -20,6 +20,7 @@
 
 import java.io.Closeable;
 import java.math.BigDecimal;
+import java.nio.charset.StandardCharsets;
 import org.apache.pinot.spi.utils.BigDecimalUtils;
 import org.apache.pinot.spi.utils.hash.MurmurHashFunctions;
 
@@ -63,7 +64,7 @@ default byte[] getUnpaddedBytes(int index, int numBytesPerValue, byte[] buffer)
    */
   default String getUnpaddedString(int index, int numBytesPerValue, byte[] buffer) {
     int length = readUnpaddedBytes(index, numBytesPerValue, buffer);
-    return new String(buffer, 0, length);
+    return new String(buffer, 0, length, StandardCharsets.UTF_8);
   }
 
   /**

pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readerwriter/FixedByteValueReaderWriterTest.java

@@ -74,4 +74,40 @@ public void testFixedByteValueReaderWriter(int maxStringLength, int configuredMa
       }
     }
   }
+
+  @Test(dataProvider = "params")
+  public void testFixedByteValueReaderWriterNonAscii(int maxStringLength, int configuredMaxLength, ByteOrder byteOrder)
+      throws IOException {
+    byte[] bytes = new byte[configuredMaxLength];
+    // Use a multi-byte UTF-8 character (é = 0xC3 0xA9)
+    byte[] nonAsciiChar = "é".getBytes(StandardCharsets.UTF_8);
+
+    try (PinotDataBuffer buffer = PinotDataBuffer.allocateDirect(configuredMaxLength * 1000L, byteOrder,
+        "testFixedByteValueReaderWriterNonAscii")) {
+      FixedByteValueReaderWriter readerWriter = new FixedByteValueReaderWriter(buffer);
+      List<String> inputs = new ArrayList<>(1000);
+
+      for (int i = 0; i < 1000; i++) {
+        // number of *characters* to write
+        int charCount = ThreadLocalRandom.current().nextInt(maxStringLength);
+        int byteCount = charCount * nonAsciiChar.length;
+        if (byteCount > configuredMaxLength) {
+          byteCount = configuredMaxLength - (configuredMaxLength % nonAsciiChar.length); // fit whole chars
+          charCount = byteCount / nonAsciiChar.length;
+        }
+
+        Arrays.fill(bytes, (byte) 0);
+        for (int pos = 0; pos < byteCount; pos += nonAsciiChar.length) {
+          System.arraycopy(nonAsciiChar, 0, bytes, pos, nonAsciiChar.length);
+        }
+
+        readerWriter.writeBytes(i, configuredMaxLength, bytes);
+        inputs.add("é".repeat(charCount));
+      }
+
+      for (int i = 0; i < 1000; i++) {
+        assertEquals(readerWriter.getUnpaddedString(i, configuredMaxLength, bytes), inputs.get(i));
+      }
+    }
+  }
 }