Merge pull request #1665 from Kotlin/csv-charsets

Custom Csv charsets support

Author: Jolanrensen

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/common.kt

@@ -1,5 +1,6 @@
 package org.jetbrains.kotlinx.dataframe.io
 
+import org.apache.commons.io.ByteOrderMark
 import org.apache.commons.io.input.BOMInputStream
 import org.jetbrains.kotlinx.dataframe.AnyFrame
 import org.jetbrains.kotlinx.dataframe.DataFrame
@@ -88,5 +89,12 @@ public fun asUrl(fileOrUrl: String): URL =
 public fun InputStream.skippingBomCharacters(): InputStream =
     BOMInputStream.builder()
         .setInputStream(this)
+        .setByteOrderMarks(
+            ByteOrderMark.UTF_8,
+            ByteOrderMark.UTF_16LE,
+            ByteOrderMark.UTF_16BE,
+            ByteOrderMark.UTF_32LE,
+            ByteOrderMark.UTF_32BE,
+        )
         .setInclude(false)
         .get()

dataframe-csv/api/dataframe-csv.api

@@ -3,6 +3,7 @@
 package org.jetbrains.kotlinx.dataframe.documentationCsv
 
 import org.jetbrains.kotlinx.dataframe.DataFrame
+import org.jetbrains.kotlinx.dataframe.documentationCsv.DelimParams.CHARSET
 import org.jetbrains.kotlinx.dataframe.io.ColType
 import java.io.File
 import java.io.InputStream
@@ -83,6 +84,7 @@ internal interface CommonReadDelimDocs {
     interface DelimDocs
 
     /**
+     * @include [CHARSET]
      * @include [DelimParams.HEADER]
      * @include [DelimParams.HAS_FIXED_WIDTH_COLUMNS]
      * @include [DelimParams.FIXED_COLUMN_WIDTHS]

dataframe-csv/src/main/kotlin/org/jetbrains/kotlinx/dataframe/documentationCsv/CommonReadDelimDocs.kt

@@ -11,6 +11,7 @@ import org.jetbrains.kotlinx.dataframe.io.ColType
 import org.jetbrains.kotlinx.dataframe.io.Compression
 import org.jetbrains.kotlinx.dataframe.io.DefaultNullStringsContentLink
 import org.jetbrains.kotlinx.dataframe.io.QuoteMode
+import java.nio.charset.Charset
 
 /**
  * Contains both the default values of csv/tsv parameters and the parameter KDocs.
@@ -20,29 +21,36 @@ internal object DelimParams {
 
     /**
      * @param path The file path to read.
+     *   Use [charset\] to specify the encoding.
      *   Can also be compressed as `.gz` or `.zip`, see [Compression][Compression].
      */
     interface PATH_READ
 
     /**
      * @param file The file to read.
+     *   Use [charset\] to specify the encoding.
      *   Can also be compressed as `.gz` or `.zip`, see [Compression][Compression].
      */
     interface FILE_READ
 
     /**
      * @param url The URL from which to fetch the data.
+     *   Use [charset\] to specify the encoding.
      *   Can also be compressed as `.gz` or `.zip`, see [Compression][Compression].
      */
     interface URL_READ
 
     /**
      * @param fileOrUrl The file path or URL to read the data from.
+     *   Use [charset\] to specify the encoding.
      *   Can also be compressed as `.gz` or `.zip`, see [Compression][Compression].
      */
     interface FILE_OR_URL_READ
 
-    /** @param inputStream Represents the file to read. */
+    /**
+     * @param inputStream Represents the file to read.
+     *   Use [charset\] to specify the encoding.
+     */
     interface INPUT_STREAM_READ
 
     /** @param text The raw data to read in the form of a [String]. */
@@ -57,6 +65,15 @@ internal object DelimParams {
     /** @param writer The [Appendable] to write to. */
     interface WRITER_WRITE
 
+    /**
+     * @param charset The [character set][java.nio.charset.Charset] the input is encoded in.
+     *   Default: `null`
+     *
+     *   If `null`, the Charset will be read from the BOM of the provided input,
+     *   defaulting to [UTF-8][Charsets.UTF_8] if no BOM is found.
+     */
+    val CHARSET: Charset? = null
+
     /**
      * @param delimiter The field delimiter character. Default: ','.
      *

dataframe-csv/src/main/kotlin/org/jetbrains/kotlinx/dataframe/documentationCsv/DelimParams.kt

@@ -22,6 +22,7 @@ import io.deephaven.csv.util.CsvReaderException
 import kotlinx.datetime.LocalDate
 import kotlinx.datetime.LocalDateTime
 import kotlinx.datetime.LocalTime
+import org.apache.commons.io.input.BOMInputStream
 import org.jetbrains.kotlinx.dataframe.DataColumn
 import org.jetbrains.kotlinx.dataframe.DataFrame
 import org.jetbrains.kotlinx.dataframe.DataRow
@@ -34,6 +35,7 @@ import org.jetbrains.kotlinx.dataframe.api.tryParse
 import org.jetbrains.kotlinx.dataframe.columns.ValueColumn
 import org.jetbrains.kotlinx.dataframe.documentationCsv.DelimParams.ADJUST_CSV_SPECS
 import org.jetbrains.kotlinx.dataframe.documentationCsv.DelimParams.ALLOW_MISSING_COLUMNS
+import org.jetbrains.kotlinx.dataframe.documentationCsv.DelimParams.CHARSET
 import org.jetbrains.kotlinx.dataframe.documentationCsv.DelimParams.COL_TYPES
 import org.jetbrains.kotlinx.dataframe.documentationCsv.DelimParams.COMPRESSION
 import org.jetbrains.kotlinx.dataframe.documentationCsv.DelimParams.FIXED_COLUMN_WIDTHS
@@ -61,6 +63,7 @@ import java.io.InputStream
 import java.math.BigDecimal
 import java.math.BigInteger
 import java.net.URL
+import java.nio.charset.Charset
 import kotlin.reflect.KType
 import kotlin.reflect.full.withNullability
 import kotlin.reflect.typeOf
@@ -72,6 +75,7 @@ import kotlinx.datetime.Instant as DeprecatedInstant
  * Implementation to read delimiter-separated data from an [InputStream] based on the Deephaven CSV library.
  *
  * @include [INPUT_STREAM_READ]
+ * @include [CHARSET]
  * @param delimiter The field delimiter character. The default is ',' for CSV, '\t' for TSV.
  * @include [HEADER]
  * @include [COL_TYPES]
@@ -94,6 +98,7 @@ internal fun readDelimImpl(
     inputStream: InputStream,
     delimiter: Char,
     header: List<String>,
+    charset: Charset?,
     hasFixedWidthColumns: Boolean,
     fixedColumnWidths: List<Int>,
     colTypes: Map<String, ColType>,
@@ -139,10 +144,18 @@ internal fun readDelimImpl(
     val csvReaderResult = inputStream.useDecompressed(compression) { decompressedInputStream ->
         // read the csv
         try {
+            val deBommedInputString = decompressedInputStream.skippingBomCharacters()
+
+            // choose charset like: provided? -> from BOM? -> UTF-8
+            val streamCharset = charset
+                ?: (deBommedInputString as? BOMInputStream)?.bom?.let { Charset.forName(it.charsetName) }
+                ?: Charsets.UTF_8
+
             @Suppress("ktlint:standard:comment-wrapping")
             CsvReader.read(
                 /* specs = */ csvSpecs,
-                /* stream = */ decompressedInputStream.skippingBomCharacters(),
+                /* stream = */ deBommedInputString,
+                /* streamCharset = */ streamCharset,
                 /* sinkFactory = */ ListSink.SINK_FACTORY,
             )
         } catch (e: CsvReaderException) {

dataframe-csv/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/readDelim.kt

@@ -48,7 +48,7 @@ internal fun writeDelimImpl(
         setCommentMarker(commentChar)
         setHeaderComments(*headerComments.toTypedArray())
     }.let { adjustCsvFormat(it, it) }
-        .build()
+        .get()
 
     // let the format handle the writing, only converting AnyRow and AnyFrame to JSON
     format.print(writer).use { printer ->

dataframe-csv/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeDelim.kt

@@ -19,7 +19,7 @@ public class CsvDeephaven(private val delimiter: Char = DelimParams.CSV_DELIMITE
         DataFrame.readCsv(file = file, header = header, delimiter = delimiter)
 
     override fun readDataFrame(path: Path, header: List<String>): DataFrame<*> =
-        DataFrame.readCsv(path = path, header = header, delimiter = delimiter)
+        DataFrame.readCsv(path = path, delimiter = delimiter, header = header)
 
     override fun acceptsExtension(ext: String): Boolean = ext == "csv"