GetColumnsFromCsvFileJob: use join/split instead of unicde_escape encoding

tests/job_tests/text/csv.py

@@ -0,0 +1,32 @@
+import tempfile
+
+from sisyphus import setup_path
+
+from i6_core.text.csv import GetColumnsFromCsvFileJob
+from i6_core.util import uopen
+
+Path = setup_path(__package__)
+
+
+def test_get_columns_from_csv_file_job():
+    with tempfile.TemporaryDirectory() as tmpdir:
+        from sisyphus import gs
+
+        gs.WORK_DIR = tmpdir
+
+        job = GetColumnsFromCsvFileJob(csv_file=Path("files/csv/input_file.csv"), columns=[0, 1])
+
+        job._sis_setup_directory()
+        job.run()
+
+        # Column 0
+        with uopen(Path("files/csv/out_col_0.txt").get_path(), "rt") as f:
+            expected_out = f.readlines()
+        with uopen(job.out_column_values[0].get_path(), "rt") as f:
+            assert expected_out == f.readlines()
+
+        # Column 1
+        with uopen(Path("files/csv/out_col_1.txt").get_path(), "rt") as f:
+            expected_out = f.readlines()
+        with uopen(job.out_column_values[1].get_path(), "rt") as f:
+            assert expected_out == f.readlines()

tests/job_tests/text/files/csv/input_file.csv

@@ -0,0 +1,8 @@
+normal,input
+un dólar,hello
+"this is
+a test",world
+"this is'""
+another test",bye
+哔哩哔哩,哔哩哔哩
+\,\n

tests/job_tests/text/files/csv/out_col_0.txt

@@ -0,0 +1,6 @@
+normal
+un dólar
+this is\na test
+this is'\"\nanother test
+哔哩哔哩
+\\

tests/job_tests/text/files/csv/out_col_1.txt

@@ -0,0 +1,6 @@
+input
+hello
+world
+bye
+哔哩哔哩
+\\n

text/csv.py

@@ -1,6 +1,7 @@
 __all__ = ["GetColumnsFromCsvFileJob"]
 
 import csv
+import json
 
 from sisyphus import Job, Task, tk
 
@@ -12,6 +13,16 @@ class GetColumnsFromCsvFileJob(Job):
     Dumps the values of a given set of columns from a csv file onto separate text files.
     The csv file must have been previously dumped with :funcref:`csv.writer` and :funcref:`csv.writerow`.
 
+    **DISCLAIMER: the output for each column is written via `json.dumps` to escape conflictive characters like newlines.
+    `json.dumps` might escape additional tokens of its own, like double quotes.
+    The job doesn't do anything in this case, and additional postprocessing is left to the user.**
+
+    The job uses the default `csv.reader` parameters,
+    which are expected to be compatible with the default `csv.writer` parameters.
+    For instance, the contents of a column can contain the csv delimiter or newlines.
+    In this case, the start/end of each cell in a column is properly marked
+    (see e.g. https://docs.python.org/3/library/csv.html#csv.Dialect.quotechar).
+
     The i-th output file contains the i-th column.
     """
 
@@ -39,5 +50,9 @@ def run(self):
             in_csv = csv.reader(f_in, delimiter=self.delimiter)
             for csv_line in in_csv:
                 for column in self.columns:
-                    # Encoding with unicode_escape allows special characters like "\n" to be printed as intended.
-                    opened_outs[column].write(f"{csv_line[column].encode('unicode_escape').decode('utf-8')}\n")
+                    # json.dumps helps escaping conflictive characters, e.g. newlines.
+                    escaped_csv_col = json.dumps(csv_line[column], ensure_ascii=False)
+
+                    # Remove artificial start/end quotes added by json.dumps.
+                    assert escaped_csv_col.startswith('"') and escaped_csv_col.endswith('"')
+                    opened_outs[column].write(f"{escaped_csv_col[1:-1]}\n")