diff --git a/tests/job_tests/text/csv.py b/tests/job_tests/text/csv.py new file mode 100644 index 00000000..5fc5af41 --- /dev/null +++ b/tests/job_tests/text/csv.py @@ -0,0 +1,32 @@ +import tempfile + +from sisyphus import setup_path + +from i6_core.text.csv import GetColumnsFromCsvFileJob +from i6_core.util import uopen + +Path = setup_path(__package__) + + +def test_get_columns_from_csv_file_job(): + with tempfile.TemporaryDirectory() as tmpdir: + from sisyphus import gs + + gs.WORK_DIR = tmpdir + + job = GetColumnsFromCsvFileJob(csv_file=Path("files/csv/input_file.csv"), columns=[0, 1]) + + job._sis_setup_directory() + job.run() + + # Column 0 + with uopen(Path("files/csv/out_col_0.txt").get_path(), "rt") as f: + expected_out = f.readlines() + with uopen(job.out_column_values[0].get_path(), "rt") as f: + assert expected_out == f.readlines() + + # Column 1 + with uopen(Path("files/csv/out_col_1.txt").get_path(), "rt") as f: + expected_out = f.readlines() + with uopen(job.out_column_values[1].get_path(), "rt") as f: + assert expected_out == f.readlines() diff --git a/tests/job_tests/text/files/csv/input_file.csv b/tests/job_tests/text/files/csv/input_file.csv new file mode 100644 index 00000000..9b52d3ef --- /dev/null +++ b/tests/job_tests/text/files/csv/input_file.csv @@ -0,0 +1,8 @@ +normal,input +un dólar,hello +"this is +a test",world +"this is'"" +another test",bye +哔哩哔哩,哔哩哔哩 +\,\n diff --git a/tests/job_tests/text/files/csv/out_col_0.txt b/tests/job_tests/text/files/csv/out_col_0.txt new file mode 100644 index 00000000..8fbbf58e --- /dev/null +++ b/tests/job_tests/text/files/csv/out_col_0.txt @@ -0,0 +1,6 @@ +normal +un dólar +this is\na test +this is'\"\nanother test +哔哩哔哩 +\\ diff --git a/tests/job_tests/text/files/csv/out_col_1.txt b/tests/job_tests/text/files/csv/out_col_1.txt new file mode 100644 index 00000000..2b5f149c --- /dev/null +++ b/tests/job_tests/text/files/csv/out_col_1.txt @@ -0,0 +1,6 @@ +input +hello +world +bye +哔哩哔哩 +\\n diff --git a/text/csv.py b/text/csv.py index 606c89f7..7c3fd12d 100644 --- a/text/csv.py +++ b/text/csv.py @@ -1,6 +1,7 @@ __all__ = ["GetColumnsFromCsvFileJob"] import csv +import json from sisyphus import Job, Task, tk @@ -12,6 +13,16 @@ class GetColumnsFromCsvFileJob(Job): Dumps the values of a given set of columns from a csv file onto separate text files. The csv file must have been previously dumped with :funcref:`csv.writer` and :funcref:`csv.writerow`. + **DISCLAIMER: the output for each column is written via `json.dumps` to escape conflictive characters like newlines. + `json.dumps` might escape additional tokens of its own, like double quotes. + The job doesn't do anything in this case, and additional postprocessing is left to the user.** + + The job uses the default `csv.reader` parameters, + which are expected to be compatible with the default `csv.writer` parameters. + For instance, the contents of a column can contain the csv delimiter or newlines. + In this case, the start/end of each cell in a column is properly marked + (see e.g. https://docs.python.org/3/library/csv.html#csv.Dialect.quotechar). + The i-th output file contains the i-th column. """ @@ -39,5 +50,9 @@ def run(self): in_csv = csv.reader(f_in, delimiter=self.delimiter) for csv_line in in_csv: for column in self.columns: - # Encoding with unicode_escape allows special characters like "\n" to be printed as intended. - opened_outs[column].write(f"{csv_line[column].encode('unicode_escape').decode('utf-8')}\n") + # json.dumps helps escaping conflictive characters, e.g. newlines. + escaped_csv_col = json.dumps(csv_line[column], ensure_ascii=False) + + # Remove artificial start/end quotes added by json.dumps. + assert escaped_csv_col.startswith('"') and escaped_csv_col.endswith('"') + opened_outs[column].write(f"{escaped_csv_col[1:-1]}\n")