Skip to content

Commit 014bb80

Browse files
ChristianGruenChristianGruen
committed
CSV parsing: delimiter → separator. qtspecs#2745
1 parent 81bbd43 commit 014bb80

7 files changed

Lines changed: 640 additions & 662 deletions

File tree

‎basex-core/src/main/java/org/basex/build/csv/CsvOptions.java‎

Lines changed: 10 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,8 @@
1717
* @author Christian Gruen
1818
*/
1919
public class CsvOptions extends Options {
20-
/** Option: separator. Ignored if {@code FIELD_DELIMITER} is assigned. */
20+
/** Option: separator. */
2121
public static final StringOption SEPARATOR = new StringOption("separator", ",");
22-
/** Option: field delimiter. */
23-
public static final StringOption FIELD_DELIMITER = new StringOption("field-delimiter");
2422
/** Option: format. */
2523
public static final EnumOption<CsvFormat> FORMAT = new EnumOption<>("format", CsvFormat.DIRECT);
2624
/** Option: header. */
@@ -31,8 +29,6 @@ public class CsvOptions extends Options {
3129
public static final BooleanOption LAX = new BooleanOption("lax", true);
3230
/** Option: parse quotes. */
3331
public static final BooleanOption QUOTES = new BooleanOption("quotes", true);
34-
/** Option: row delimiter. */
35-
public static final StringOption ROW_DELIMITER = new StringOption("row-delimiter", "\n");
3632
/** Option: quote character. */
3733
public static final StringOption QUOTE_CHARACTER = new StringOption("quote-character", "\"");
3834
/** Option: trim whitespace. */
@@ -107,35 +103,20 @@ public synchronized void assign(final Item name, final Value value, final InputI
107103
* @throws QueryException query exception
108104
*/
109105
private void validate(final InputInfo info) throws QueryException {
110-
final StringOption separator = get(FIELD_DELIMITER) != null ? FIELD_DELIMITER : SEPARATOR;
111-
final int f = fieldDelimiter(), r = rowDelimiter(), q = quoteCharacter();
112-
final StringOption option = f == -1 ? separator : r == -1 ? ROW_DELIMITER : q == -1 ?
113-
QUOTE_CHARACTER : null;
106+
final int s = separator(), q = quoteCharacter();
107+
final StringOption option = s == -1 ? SEPARATOR : q == -1 ? QUOTE_CHARACTER : null;
114108
if(option != null) throw CSV_SINGLECHAR_X_X.get(info, option.name(), get(option));
115-
if(f == q || r == f || q == r) throw CSV_DELIMITER_X.get(info,
116-
get(f == q || r == f ? separator : QUOTE_CHARACTER));
109+
if(s == q) throw CSV_DELIMITER_X.get(info, get(SEPARATOR));
117110
}
118111

119112
/**
120-
* Returns the separator character or {@code -1} if character is invalid.
121-
* @return field delimiter
113+
* Returns the separator character or {@code -1} if the character is invalid.
114+
* @return separator
122115
*/
123-
public int fieldDelimiter() {
124-
String sep = get(FIELD_DELIMITER);
125-
if(sep == null) {
126-
sep = get(SEPARATOR);
127-
final CsvSep s = Enums.get(CsvSep.class, sep);
128-
if(s != null) return s.sep;
129-
}
130-
return checkCodepoint(sep);
131-
}
132-
133-
/**
134-
* Returns the row delimiter character or {@code -1} if character is invalid.
135-
* @return row delimiter
136-
*/
137-
public int rowDelimiter() {
138-
return checkCodepoint(get(ROW_DELIMITER));
116+
public int separator() {
117+
final String sep = get(SEPARATOR);
118+
final CsvSep s = Enums.get(CsvSep.class, sep);
119+
return s != null ? s.sep : checkCodepoint(sep);
139120
}
140121

141122
/**

‎basex-core/src/main/java/org/basex/build/csv/CsvW3ArraysOptions.java‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,11 @@
1717
* @author Gunther Rademacher
1818
*/
1919
public class CsvW3ArraysOptions extends Options {
20-
/** parse-csv option field-delimiter. */
21-
public static final StringOption FIELD_DELIMITER = new StringOption("field-delimiter", ",");
22-
/** parse-csv option row-delimiter. */
23-
public static final StringOption ROW_DELIMITER = new StringOption("row-delimiter", "\n");
24-
/** parse-csv option quote-character. */
20+
/** csv-to-arrays option separator. */
21+
public static final StringOption SEPARATOR = new StringOption("separator", ",");
22+
/** csv-to-arrays option quote-character. */
2523
public static final StringOption QUOTE_CHARACTER = new StringOption("quote-character", "\"");
26-
/** parse-csv option trim-whitespace. */
24+
/** csv-to-arrays option trim-whitespace. */
2725
public static final BooleanOption TRIM_WHITESPACE = new BooleanOption("trim-whitespace", false);
2826

2927
/**
@@ -35,7 +33,7 @@ public class CsvW3ArraysOptions extends Options {
3533
*/
3634
public CsvParserOptions finish(final InputInfo ii, final CsvFormat format) throws QueryException {
3735
final IntSet delim = new IntSet();
38-
for(final StringOption opt : Arrays.asList(FIELD_DELIMITER, ROW_DELIMITER, QUOTE_CHARACTER)) {
36+
for(final StringOption opt : Arrays.asList(SEPARATOR, QUOTE_CHARACTER)) {
3937
final String val = get(opt);
4038
if(val.codePointCount(0, val.length()) != 1)
4139
throw CSV_SINGLECHAR_X_X.get(ii, opt.name(), val);
@@ -45,8 +43,7 @@ public CsvParserOptions finish(final InputInfo ii, final CsvFormat format) throw
4543

4644
final CsvParserOptions copts = new CsvParserOptions();
4745
copts.set(CsvOptions.FORMAT, format);
48-
copts.set(CsvOptions.FIELD_DELIMITER, get(FIELD_DELIMITER));
49-
copts.set(CsvOptions.ROW_DELIMITER, get(ROW_DELIMITER));
46+
copts.set(CsvOptions.SEPARATOR, get(SEPARATOR));
5047
copts.set(CsvOptions.QUOTE_CHARACTER, get(QUOTE_CHARACTER));
5148
copts.set(CsvOptions.TRIM_WHITESPACE, get(TRIM_WHITESPACE));
5249
copts.set(CsvOptions.STRICT_QUOTING, true);

‎basex-core/src/main/java/org/basex/io/parse/csv/CsvParser.java‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,8 @@ public final class CsvParser {
2525
private final boolean header;
2626
/** Backslash flag. */
2727
private final boolean backslashes;
28-
/** Field delimiter (see {@link CsvOptions#FIELD_DELIMITER}). */
29-
private final int fieldDelimiter;
30-
/** Row delimiter (see {@link CsvOptions#ROW_DELIMITER}). */
31-
private final int rowDelimiter;
28+
/** Separator (see {@link CsvOptions#SEPARATOR}). */
29+
private final int separator;
3230
/** Quote character (see {@link CsvOptions#QUOTE_CHARACTER}). */
3331
private final int quoteCharacter;
3432
/** Parse quotes. */
@@ -63,8 +61,7 @@ public CsvParser(final TextInput input, final CsvParserOptions opts, final CsvCo
6361
this.input = input;
6462
this.conv = conv;
6563
header = opts.get(CsvOptions.HEADER) == Bln.TRUE;
66-
fieldDelimiter = opts.fieldDelimiter();
67-
rowDelimiter = opts.rowDelimiter();
64+
separator = opts.separator();
6865
quoteCharacter = opts.quoteCharacter();
6966
strictQuoting = opts.get(CsvOptions.STRICT_QUOTING);
7067
quotes = strictQuoting || opts.get(CsvOptions.QUOTES);
@@ -96,7 +93,7 @@ public void parse(final InputInfo ii) throws QueryException, IOException {
9693
ch = input.read();
9794
if(ch != quoteCharacter) {
9895
quoted = false;
99-
if(strictQuoting && ch != fieldDelimiter && ch != rowDelimiter && ch != -1)
96+
if(strictQuoting && ch != separator && ch != '\n' && ch != -1)
10097
throw QueryError.CSV_QUOTING_X.get(ii, new TokenBuilder().add(
10198
quoteCharacter).add(entry).add(quoteCharacter).add(ch));
10299
continue;
@@ -117,11 +114,11 @@ public void parse(final InputInfo ii) throws QueryException, IOException {
117114
if(ch != quoteCharacter || backslashes) add(entry, quoteCharacter);
118115
continue;
119116
}
120-
} else if(ch == fieldDelimiter) {
117+
} else if(ch == separator) {
121118
// parse separator
122119
record(entry, false, false);
123120
first = false;
124-
} else if(ch == rowDelimiter) {
121+
} else if(ch == '\n') {
125122
// parse newline
126123
record(entry, false, true);
127124
first = true;

‎basex-core/src/main/java/org/basex/io/serial/csv/CsvSerializer.java‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,11 @@ public abstract class CsvSerializer extends StandardSerializer {
2323
/** CSV options. */
2424
final CsvOptions copts;
2525
/** Separator. */
26-
final int fieldDelimiter;
26+
final int separator;
2727
/** Generate quotes. */
2828
final boolean quotes;
2929
/** Generate backslashes. */
3030
final boolean backslashes;
31-
/** Row delimiter (see {@link CsvOptions#ROW_DELIMITER}). */
32-
private final int rowDelimiter;
3331
/** Quote character (see {@link CsvOptions#QUOTE_CHARACTER}). */
3432
private final int quoteCharacter;
3533
/** Select columns (see {@link CsvOptions#SELECT_COLUMNS}). */
@@ -68,8 +66,7 @@ public static Serializer get(final OutputStream os, final SerializerOptions so)
6866
copts = sopts.get(SerializerOptions.CSV);
6967
quotes = copts.get(CsvOptions.QUOTES);
7068
backslashes = copts.get(CsvOptions.BACKSLASHES);
71-
fieldDelimiter = copts.fieldDelimiter();
72-
rowDelimiter = copts.rowDelimiter();
69+
separator = copts.separator();
7370
quoteCharacter = copts.quoteCharacter();
7471
selectColumns = copts.get(CsvOptions.SELECT_COLUMNS);
7572
maxCol = -1;
@@ -116,7 +113,7 @@ final void record(final TokenList entries, final boolean reset) throws IOExcepti
116113
field(f++, val == null ? Token.EMPTY : val);
117114
}
118115
}
119-
out.print(rowDelimiter);
116+
out.print('\n');
120117
if(reset) entries.reset();
121118
}
122119

@@ -128,11 +125,11 @@ final void record(final TokenList entries, final boolean reset) throws IOExcepti
128125
*/
129126
final void field(final int seqNo, final byte[] value) throws IOException {
130127
// print fields, skip trailing empty contents
131-
if(seqNo != 0) out.print(fieldDelimiter);
128+
if(seqNo != 0) out.print(separator);
132129

133130
byte[] txt = value != null ? value : Token.EMPTY;
134131
if(form != null) txt = normalize(txt, form);
135-
final boolean delim = contains(txt, fieldDelimiter) || contains(txt, rowDelimiter);
132+
final boolean delim = contains(txt, separator) || contains(txt, '\n');
136133
final boolean special = contains(txt, '\r') || contains(txt, '\t')
137134
|| contains(txt, quoteCharacter);
138135
if(delim || special || backslashes && contains(txt, '\\')) {
@@ -145,12 +142,12 @@ final void field(final int seqNo, final byte[] value) throws IOException {
145142
while(tp.more()) {
146143
final int cp = tp.next();
147144
if(backslashes) {
148-
if(cp == '\n') tb.add("\\").add(fieldDelimiter == '\n' ? "n" : cp);
145+
if(cp == '\n') tb.add("\\").add(separator == '\n' ? "n" : cp);
149146
else if(cp == '\r') tb.add("\\r");
150147
else if(cp == '\t') tb.add("\\t");
151148
else if(cp == quoteCharacter) tb.add("\\").add(cp);
152149
else if(cp == '\\') tb.add("\\\\");
153-
else if(cp == fieldDelimiter && !quotes) tb.add('\\').add(cp);
150+
else if(cp == separator && !quotes) tb.add('\\').add(cp);
154151
else tb.add(cp);
155152
} else {
156153
if(cp == quoteCharacter) tb.add(quoteCharacter);

‎basex-core/src/main/java/org/basex/query/QueryError.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -566,7 +566,7 @@ public enum QueryError {
566566
/** Error code. */
567567
CSV_SINGLECHAR_X_X(FOCV, 2, "The value of '%' is not a single character: '%'."),
568568
/** Error code. */
569-
CSV_DELIMITER_X(FOCV, 3, "Duplicate CSV delimiter error: %."),
569+
CSV_DELIMITER_X(FOCV, 2, "The same character is used for more than one CSV option: %."),
570570
/** Error code. */
571571
CSV_COLUMNNAME_X(FOCV, 4, "Argument supplied is not a known column name: %."),
572572

0 commit comments

Comments
 (0)