Skip to content

Decode multiline CSV. #770

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
blackwinter wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

Decode multiline CSV. #770

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7dbdd75
Add CSV decoder tests for invalid and multiline data. (#769)
blackwinter May 15, 2026
dae6a4d
Decode multiline CSV. (#769)
blackwinter May 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 36 additions & 35 deletions metafacture-csv/src/main/java/org/metafacture/csv/CsvDecoder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package org.metafacture.csv;

import org.metafacture.framework.FluxCommand;
import org.metafacture.framework.MetafactureException;
import org.metafacture.framework.StreamReceiver;
import org.metafacture.framework.annotations.Description;
import org.metafacture.framework.annotations.In;
Expand All @@ -31,7 +32,6 @@

import java.io.IOException;
import java.io.StringReader;
import java.util.List;

/**
* Decodes lines of CSV files. First line may be interpreted as header.
Expand Down Expand Up @@ -91,50 +91,51 @@ private void initializeCsvParser() {
@Override
public void process(final String string) {
assert !isClosed();
final String[] parts = parseCsv(string);
if (hasHeader) {
if (header.length == 0) {
header = parts;
}
else if (parts.length == header.length) {
getReceiver().startRecord(String.valueOf(++count));
for (int i = 0; i < parts.length; ++i) {
getReceiver().literal(header[i], parts[i]);
try (
StringReader sr = new StringReader(string);
CSVReader reader = new CSVReaderBuilder(sr).withCSVParser(parser).build()
) {
String[] parts;
while ((parts = parseCsv(reader)) != null) {
if (hasHeader) {
if (header.length == 0) {
header = parts;
}
else if (parts.length == header.length) {
getReceiver().startRecord(String.valueOf(++count));
for (int i = 0; i < parts.length; ++i) {
getReceiver().literal(header[i], parts[i]);
}
getReceiver().endRecord();
}
else {
throw new IllegalArgumentException(
String.format(
"wrong number of columns (expected %s, was %s) in input line: %s",
header.length, parts.length, string));
}
}
else {
getReceiver().startRecord(String.valueOf(++count));
for (int i = 0; i < parts.length; ++i) {
getReceiver().literal(String.valueOf(i), parts[i]);
}
getReceiver().endRecord();
}
getReceiver().endRecord();
}
else {
throw new IllegalArgumentException(
String.format(
"wrong number of columns (expected %s, was %s) in input line: %s",
header.length, parts.length, string));
}
}
else {
getReceiver().startRecord(String.valueOf(++count));
for (int i = 0; i < parts.length; ++i) {
getReceiver().literal(String.valueOf(i), parts[i]);
}
getReceiver().endRecord();
catch (final IOException e) {
throw new MetafactureException(e);
}
}

private String[] parseCsv(final String csv) {
String[] parts = new String[0];
private String[] parseCsv(final CSVReader reader) {
try {
final CSVReader reader = new CSVReaderBuilder(new StringReader(csv))
.withCSVParser(parser)
.build();
final List<String[]> lines = reader.readAll();
if (lines.size() > 0) {
parts = lines.get(0);
}
reader.close();
return reader.readNext();
}
catch (final IOException | CsvException e) {
e.printStackTrace();
throw new MetafactureException(e);
}
return parts;
}

/**
Expand Down
68 changes: 68 additions & 0 deletions metafacture-csv/src/test/java/org/metafacture/csv/CsvDecoderTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,14 @@

package org.metafacture.csv;

import org.metafacture.framework.MetafactureException;
import org.metafacture.framework.StreamReceiver;

import com.opencsv.exceptions.CsvMalformedLineException;
import org.hamcrest.CoreMatchers;
import org.hamcrest.MatcherAssert;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.InOrder;
Expand Down Expand Up @@ -78,6 +83,49 @@ public void testQuoted() {
ordered.verify(receiver).endRecord();
}

@Test
public void testInvalidSyntax() {
assertException(MetafactureException.class, CsvMalformedLineException.class,
"Unterminated quoted field at end of CSV line", "a,\"b1,b2,b3,c"); // missing closing "
}

@Test
public void testInvalidColumns() {
assertException(IllegalArgumentException.class, null,
"wrong number of columns (expected 3, was 2)", "a,b"); // missing third column
}

@Test
public void testMultilineSingleRow() {
decoder.process("a,\"b1,b2,\nb3\",c");
final InOrder ordered = Mockito.inOrder(receiver);
ordered.verify(receiver).startRecord("1");
ordered.verify(receiver).literal("h1", "a");
ordered.verify(receiver).literal("h2", "b1,b2,\nb3");
ordered.verify(receiver).literal("h3", "c");
ordered.verify(receiver).endRecord();
ordered.verifyNoMoreInteractions();
Mockito.verifyNoMoreInteractions(receiver);
}

@Test
public void testMultilineMultipleRows() {
decoder.process("a,\"b1,b2,\nb3\",c\na2,b4,c2");
final InOrder ordered = Mockito.inOrder(receiver);
ordered.verify(receiver).startRecord("1");
ordered.verify(receiver).literal("h1", "a");
ordered.verify(receiver).literal("h2", "b1,b2,\nb3");
ordered.verify(receiver).literal("h3", "c");
ordered.verify(receiver).endRecord();
ordered.verify(receiver).startRecord("2");
ordered.verify(receiver).literal("h1", "a2");
ordered.verify(receiver).literal("h2", "b4");
ordered.verify(receiver).literal("h3", "c2");
ordered.verify(receiver).endRecord();
ordered.verifyNoMoreInteractions();
Mockito.verifyNoMoreInteractions(receiver);
}

@Test
public void testTabSeparated() {

Expand Down Expand Up @@ -110,4 +158,24 @@ public void issue496_escaping() {
ordered.verify(receiver).endRecord();
}

private void assertException(final Class<? extends Throwable> exceptionClass, final Class<?> expectedClass, final String expectedMessage, final String input) {
final Throwable thrownException = Assert.assertThrows(exceptionClass, () -> decoder.process(input));
final Throwable actualException;

final String expectedSuffix;

if (expectedClass != null) {
actualException = thrownException.getCause();
Assert.assertEquals(expectedClass, actualException.getClass());

expectedSuffix = ". ";
}
else {
actualException = thrownException;
expectedSuffix = " in ";
}

MatcherAssert.assertThat(actualException.getMessage(), CoreMatchers.startsWith(expectedMessage + expectedSuffix));
}

}
Loading