[tabular] multi-line support

Author: tstack

src/base/auto_mem.hh

@@ -300,7 +300,7 @@ public:
 
     auto_buffer& append(std::string_view sv)
     {
-        if (this->ab_size + sv.length() < this->ab_capacity) {
+        if (this->ab_size + sv.length() > this->ab_capacity) {
             this->expand_by(sv.length() + 1024);
         }
         memcpy(&this->ab_buffer[this->ab_size], sv.data(), sv.length());

src/base/separated_string.cc

@@ -197,8 +197,22 @@ separated_string::iterator::update()
     bool saw_quote = false;
     const char* quote_start = nullptr;
     const char* quote_end = nullptr;
+    this->i_unterminated_quote = false;
 
     const char* p = this->i_pos;
+    // Continuation case: the prior chunk ended mid-quote and the
+    // caller stitched the next chunk on.  Begin parsing this cell
+    // already inside the quoted region — value bytes start at p,
+    // not after a leading `"`.  Only applies to the very first cell
+    // of the buffer.
+    if (this->i_parent.ss_resume && this->i_parent.ss_resume->rs_in_quote
+        && p == ss.ss_str)
+    {
+        in_quotes = true;
+        saw_quote = true;
+        quote_start = p;
+        state = OTHER;
+    }
     while (p < data_end) {
         if (!in_quotes && *p == sep_ch) {
             if (sep_ch == ' ' && p + 1 < data_end) {
@@ -320,6 +334,14 @@ separated_string::iterator::update()
     }
     this->i_next_pos = (p < data_end) ? p + 1 : data_end;
 
+    // The parse loop only exits with `in_quotes` still set when it
+    // ran off data_end without seeing a close-quote.  Surface that
+    // to the caller so they can decide whether more bytes are
+    // pending (e.g. a CSV cell whose value spans physical lines).
+    if (in_quotes) {
+        this->i_unterminated_quote = true;
+    }
+
     if (saw_quote) {
         // Use the span between the quotes.  An unterminated
         // quote takes everything up to the current position.

src/base/separated_string.hh

@@ -59,8 +59,29 @@ struct separated_string {
 
     const char* ss_str;
     size_t ss_len;
+    // Cross-buffer parser state.  Captured by `iterator::suspend()`
+    // when a buffer ends mid-row (typically a quoted cell whose
+    // closing `"` lives in a not-yet-arrived chunk) and replayed by
+    // assigning the snapshot to `ss_resume` on the next buffer.
+    // Carries:
+    //   - rs_index — the row-level cell index the next iterator
+    //     should report on its first cell, so a continuation that
+    //     finishes column 7 of 10 reports indices 7..9 instead of
+    //     restarting at 0.
+    //   - rs_in_quote — when true, the first cell starts already
+    //     inside an open quoted region: no leading `"` required,
+    //     separators stay suppressed until a closing `"`, and `""`
+    //     is still an embedded literal.  If the close never arrives,
+    //     the trailing cell again carries `unterminated_quote()` and
+    //     the caller can keep stitching.
+    struct resume_state {
+        size_t rs_index{0};
+        bool rs_in_quote{false};
+    };
+
     char ss_separator{','};
     std::optional<size_t> ss_expected_count;
+    std::optional<resume_state> ss_resume;
 
     separated_string(const char* str, size_t len) : ss_str(str), ss_len(len) {}
 
@@ -103,10 +124,17 @@ struct separated_string {
         // show" from "at data_end, natural end" so operator==(end())
         // doesn't cut iteration short.
         bool i_in_ghost{false};
+        // True when update() reached data_end while still inside a
+        // quoted region — the closing `"` was never seen because the
+        // buffer was cut short.  Only the trailing cell can ever
+        // carry it.  Callers (e.g. multi-line CSV ingest) can use it
+        // to decide whether to splice in more bytes and re-parse.
+        bool i_unterminated_quote{false};
 
         iterator(const separated_string& ss, const char* pos)
             : i_parent(ss), i_pos(pos), i_next_pos(pos), i_value_start(pos),
-              i_value_end(pos), i_index(0)
+              i_value_end(pos),
+              i_index(ss.ss_resume ? ss.ss_resume->rs_index : 0)
         {
             this->update();
         }
@@ -140,6 +168,24 @@ struct separated_string {
 
         cell_kind kind() const { return this->i_kind; }
 
+        // True only on the trailing cell when the buffer ended inside
+        // an open quoted region.  Stays false on every other cell.
+        bool unterminated_quote() const
+        {
+            return this->i_unterminated_quote;
+        }
+
+        // Snapshot the cross-buffer parser state at the current cell.
+        // Typical use: when `unterminated_quote()` returns true on the
+        // trailing cell, capture the snapshot, accumulate the cell's
+        // partial value into your own buffer, then assign the snapshot
+        // to the next buffer's `ss_resume` so the continuation iterator
+        // picks up at the right index already inside the open quote.
+        resume_state suspend() const
+        {
+            return {this->i_index, this->i_unterminated_quote};
+        }
+
         bool operator==(const iterator& other) const
         {
             return (&this->i_parent == &other.i_parent)

src/base/separated_string.tests.cc

@@ -271,6 +271,142 @@ TEST_CASE("unterminated quote captures rest of input")
     CHECK(cells[0].value == "start,here,no-close");
 }
 
+TEST_CASE("unterminated_quote flag flips only on the trailing open cell")
+{
+    auto input = std::string(R"(a,"unterminated cell)");
+    separated_string ss(input.data(), input.length());
+    std::vector<bool> flags;
+    for (auto iter = ss.begin(); iter != ss.end(); ++iter) {
+        flags.push_back(iter.unterminated_quote());
+    }
+    REQUIRE(flags.size() == 2);
+    CHECK_FALSE(flags[0]);
+    CHECK(flags[1]);
+}
+
+TEST_CASE("unterminated_quote stays false when every quote is closed")
+{
+    auto input = std::string(R"("one","two","three")");
+    separated_string ss(input.data(), input.length());
+    for (auto iter = ss.begin(); iter != ss.end(); ++iter) {
+        CHECK_FALSE(iter.unterminated_quote());
+    }
+}
+
+TEST_CASE("unterminated_quote sees through `\"\"` escapes")
+{
+    // The `""` is an embedded literal, not a close-quote — so the
+    // outer quoted region is still open at end-of-buffer.
+    auto input = std::string(R"("a""b,rest)");
+    separated_string ss(input.data(), input.length());
+    auto iter = ss.begin();
+    REQUIRE(iter != ss.end());
+    CHECK(iter.unterminated_quote());
+    ++iter;
+    CHECK(iter == ss.end());
+}
+
+TEST_CASE("ss_resume: continuation closes and emits remaining cells")
+{
+    // Caller previously parsed `"line one` (open quote), is now
+    // feeding the rest after a `\n` glue: `line two",x,y`.
+    auto input = std::string(R"(line two",x,y)");
+    separated_string ss(input.data(), input.length());
+    ss.ss_resume = separated_string::resume_state{0, true};
+    auto cells = std::vector<parsed_cell>{};
+    for (auto iter = ss.begin(); iter != ss.end(); ++iter) {
+        cells.push_back(parsed_cell{(*iter).to_string(), iter.kind()});
+        CHECK_FALSE(iter.unterminated_quote());
+    }
+    REQUIRE(cells.size() == 3);
+    CHECK(cells[0].value == "line two");
+    CHECK(cells[1].value == "x");
+    CHECK(cells[2].value == "y");
+}
+
+TEST_CASE("ss_resume: continuation still unterminated re-flags")
+{
+    // Open-quote chunk N+1: still no closing `"` in this buffer
+    // either.  Caller will need to keep stitching.
+    auto input = std::string("more middle text");
+    separated_string ss(input.data(), input.length());
+    ss.ss_resume = separated_string::resume_state{0, true};
+    auto iter = ss.begin();
+    REQUIRE(iter != ss.end());
+    CHECK((*iter).to_string() == "more middle text");
+    CHECK(iter.unterminated_quote());
+    ++iter;
+    CHECK(iter == ss.end());
+}
+
+TEST_CASE("ss_resume: `\"\"` inside a continuation stays embedded")
+{
+    // `""` in a continuation chunk is still a literal `"`, not a
+    // close-quote; the surrounding region remains open until a
+    // lone `"`.
+    auto input = std::string(R"(has ""embedded"" still open)");
+    separated_string ss(input.data(), input.length());
+    ss.ss_resume = separated_string::resume_state{0, true};
+    auto iter = ss.begin();
+    REQUIRE(iter != ss.end());
+    CHECK((*iter).to_string() == R"(has ""embedded"" still open)");
+    CHECK(iter.unterminated_quote());
+}
+
+TEST_CASE("ss_resume: continuation that closes immediately")
+{
+    // Edge case: the buffer is just the close-quote, separator, next
+    // cell.  The leading character closes the open region and yields
+    // an empty cell value (no bytes from this chunk belonged to the
+    // quoted run).
+    auto input = std::string(R"(",tail)");
+    separated_string ss(input.data(), input.length());
+    ss.ss_resume = separated_string::resume_state{0, true};
+    auto cells = std::vector<parsed_cell>{};
+    for (auto iter = ss.begin(); iter != ss.end(); ++iter) {
+        cells.push_back(parsed_cell{(*iter).to_string(), iter.kind()});
+    }
+    REQUIRE(cells.size() == 2);
+    CHECK(cells[0].value.empty());
+    CHECK(cells[1].value == "tail");
+}
+
+TEST_CASE("ss_resume: suspend() preserves row-level cell index")
+{
+    // Buffer 1: 3 closed cells, then a 4th that opens a quote and
+    // never closes.  suspend() snapshot should report index 3 +
+    // in-quote.
+    auto buf1 = std::string(R"(a,b,c,"open and unfinished)");
+    separated_string ss1(buf1.data(), buf1.length());
+    std::optional<separated_string::resume_state> snap;
+    for (auto iter = ss1.begin(); iter != ss1.end(); ++iter) {
+        if (iter.unterminated_quote()) {
+            snap = iter.suspend();
+        }
+    }
+    REQUIRE(snap.has_value());
+    CHECK(snap->rs_index == 3);
+    CHECK(snap->rs_in_quote);
+
+    // Buffer 2: continuation closes the cell, then two more.  Indices
+    // emitted should be 3, 4, 5 — the snapshot's row-level position
+    // carried across the buffer boundary.
+    auto buf2 = std::string(R"(rest of cell",d,e)");
+    separated_string ss2(buf2.data(), buf2.length());
+    ss2.ss_resume = *snap;
+    std::vector<size_t> indices;
+    std::vector<std::string> values;
+    for (auto iter = ss2.begin(); iter != ss2.end(); ++iter) {
+        indices.push_back(iter.index());
+        values.push_back((*iter).to_string());
+    }
+    REQUIRE(indices.size() == 3);
+    CHECK(indices == std::vector<size_t>{3, 4, 5});
+    CHECK(values[0] == "rest of cell");
+    CHECK(values[1] == "d");
+    CHECK(values[2] == "e");
+}
+
 TEST_CASE("newlines inside a quoted cell are preserved verbatim")
 {
     auto cells = tokenize("\"line1\nline2\",next");