Skip to content

Commit bed3541

Browse files
tstacktstack
committed
[db] use sparse cell container for DB view
1 parent 283ef00 commit bed3541

9 files changed

Lines changed: 332 additions & 15 deletions

‎src/base/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ add_library(
8383
result.h
8484
short_alloc.h
8585
small_string_map.hh
86+
sparse_cursor_container.hh
8687
snippet_highlighters.hh
8788
string_attr_type.hh
8889
strnatcmp.h
@@ -112,6 +113,7 @@ add_executable(
112113
test_base
113114
attr_line.tests.cc
114115
cell_container.tests.cc
116+
sparse_cursor_container.tests.cc
115117
fs_util.tests.cc
116118
humanize.file_size.tests.cc
117119
humanize.network.tests.cc

‎src/base/Makefile.am‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ noinst_HEADERS = \
7777
short_alloc.h \
7878
small_string_map.hh \
7979
snippet_highlighters.hh \
80+
sparse_cursor_container.hh \
8081
string_attr_type.hh \
8182
string_util.hh \
8283
strnatcmp.h \
@@ -134,6 +135,7 @@ test_base_SOURCES = \
134135
lnav.gzip.tests.cc \
135136
math_util.tests.cc \
136137
small_string_map.tests.cc \
138+
sparse_cursor_container.tests.cc \
137139
string_util.tests.cc \
138140
test_base.cc
139141

‎src/base/sparse_cursor_container.hh‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
/**
2+
* Copyright (c) 2026, Timothy Stack
3+
*
4+
* All rights reserved.
5+
*
6+
* Redistribution and use in source and binary forms, with or without
7+
* modification, are permitted provided that the following conditions are met:
8+
*
9+
* * Redistributions of source code must retain the above copyright notice, this
10+
* list of conditions and the following disclaimer.
11+
* * Redistributions in binary form must reproduce the above copyright notice,
12+
* this list of conditions and the following disclaimer in the documentation
13+
* and/or other materials provided with the distribution.
14+
* * Neither the name of Timothy Stack nor the names of its contributors
15+
* may be used to endorse or promote products derived from this software
16+
* without specific prior written permission.
17+
*
18+
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
19+
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20+
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21+
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
22+
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23+
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24+
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
25+
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26+
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27+
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28+
*/
29+
30+
#ifndef lnav_sparse_cursor_container_hh
31+
#define lnav_sparse_cursor_container_hh
32+
33+
#include <cstddef>
34+
#include <vector>
35+
36+
#include "cell_container.hh"
37+
#include "lnav_log.hh"
38+
39+
namespace lnav {
40+
41+
/**
42+
* Index over fixed-width rows in a `cell_container` that only stores a
43+
* cursor every `BlockRows` rows. Random access resolves `row` by
44+
* jumping to the nearest stored cursor and walking forward one cell at
45+
* a time (`columns` cells per row).
46+
*
47+
* Memory: one `cell_container::cursor` (16 bytes) per block of rows,
48+
* versus 16 bytes per row for a dense `std::vector<cursor>` — a 64x
49+
* savings at the default block size for millions-of-rows result sets.
50+
*
51+
* Time: random access is amortized O(columns * BlockRows); for the DB
52+
* view with 10s of columns that's a handful of pointer chases, far
53+
* under a millisecond even for the worst case.
54+
*
55+
* Preconditions:
56+
* - all rows have the same cell count (`columns`), set once at the
57+
* first `push_row()` or explicitly via `set_columns()`.
58+
* - rows are appended in order and never removed or edited in
59+
* place; the backing `cell_container` is append-only.
60+
*/
61+
template<size_t BlockRows = 64>
62+
struct sparse_cursor_container {
63+
static_assert(BlockRows > 0, "BlockRows must be positive");
64+
static constexpr size_t BLOCK_ROWS = BlockRows;
65+
66+
using cursor = cell_container::cursor;
67+
68+
sparse_cursor_container() = default;
69+
70+
/**
71+
* Record the start of a new row. Must be called once per row
72+
* *before* pushing its cells into the `cell_container`, with the
73+
* current `end_cursor()` of that container.
74+
*/
75+
void push_row(cursor row_start)
76+
{
77+
if (this->scc_size % BlockRows == 0) {
78+
this->scc_block_starts.push_back(row_start);
79+
}
80+
++this->scc_size;
81+
}
82+
83+
/**
84+
* Configure the column count. May be left at 0 until the first
85+
* `at()` call; if 0 at that point, we infer it by walking the
86+
* first row up to the second recorded block start.
87+
*/
88+
void set_columns(size_t columns) { this->scc_columns = columns; }
89+
90+
size_t size() const { return this->scc_size; }
91+
bool empty() const { return this->scc_size == 0; }
92+
93+
cursor at(size_t row) const
94+
{
95+
require(row < this->scc_size);
96+
require(this->scc_columns > 0);
97+
98+
const auto block = row / BlockRows;
99+
const auto within_block = row % BlockRows;
100+
101+
// The stored block-start cursor was captured from
102+
// `end_cursor()` *before* the block's first cell was written,
103+
// so its offset may sit at the end of a chunk whose capacity
104+
// was already exhausted. `sync()` migrates the cursor onto
105+
// the next chunk in that case; `next()` does not, so we must
106+
// sync before walking.
107+
auto synced = this->scc_block_starts[block].sync();
108+
require(synced.has_value());
109+
auto current = synced.value();
110+
for (size_t remaining = within_block * this->scc_columns;
111+
remaining > 0;
112+
--remaining)
113+
{
114+
auto next = current.next();
115+
require(next.has_value());
116+
current = next.value();
117+
}
118+
return current;
119+
}
120+
121+
cursor operator[](size_t row) const { return this->at(row); }
122+
123+
cursor front() const { return this->at(0); }
124+
cursor back() const { return this->at(this->scc_size - 1); }
125+
126+
void clear()
127+
{
128+
this->scc_block_starts.clear();
129+
this->scc_size = 0;
130+
}
131+
132+
// For telemetry / tests only: how many block-start cursors are
133+
// actually stored.
134+
size_t block_count() const { return this->scc_block_starts.size(); }
135+
136+
std::vector<cursor> scc_block_starts;
137+
size_t scc_size{0};
138+
size_t scc_columns{0};
139+
};
140+
141+
} // namespace lnav
142+
143+
#endif

‎src/base/sparse_cursor_container.tests.cc‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
/**
2+
* Copyright (c) 2026, Timothy Stack
3+
*
4+
* All rights reserved.
5+
*
6+
* Redistribution and use in source and binary forms, with or without
7+
* modification, are permitted provided that the following conditions are met:
8+
*
9+
* * Redistributions of source code must retain the above copyright notice, this
10+
* list of conditions and the following disclaimer.
11+
* * Redistributions in binary form must reproduce the above copyright notice,
12+
* this list of conditions and the following disclaimer in the documentation
13+
* and/or other materials provided with the distribution.
14+
* * Neither the name of Timothy Stack nor the names of its contributors
15+
* may be used to endorse or promote products derived from this software
16+
* without specific prior written permission.
17+
*
18+
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
19+
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20+
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21+
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
22+
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23+
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24+
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
25+
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26+
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27+
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28+
*/
29+
30+
#include "sparse_cursor_container.hh"
31+
32+
#include "doctest/doctest.h"
33+
34+
TEST_CASE("sparse_cursor_container-basic")
35+
{
36+
constexpr size_t NCOLS = 3;
37+
constexpr size_t NROWS = 500;
38+
39+
lnav::cell_container cells;
40+
lnav::sparse_cursor_container<16> idx;
41+
idx.set_columns(NCOLS);
42+
43+
for (size_t r = 0; r < NROWS; ++r) {
44+
idx.push_row(cells.end_cursor());
45+
cells.push_int_cell(static_cast<int64_t>(r * 100 + 0));
46+
cells.push_int_cell(static_cast<int64_t>(r * 100 + 1));
47+
cells.push_int_cell(static_cast<int64_t>(r * 100 + 2));
48+
}
49+
50+
CHECK(idx.size() == NROWS);
51+
// One block cursor per 16 rows, rounded up.
52+
CHECK(idx.block_count() == (NROWS + 15) / 16);
53+
54+
// Spot-check every 37th row to cover aligned/unaligned positions.
55+
for (size_t r = 0; r < NROWS; r += 37) {
56+
auto c = idx.at(r);
57+
CHECK(c.get_type() == lnav::cell_type::CT_INTEGER);
58+
CHECK(c.get_int() == static_cast<int64_t>(r * 100 + 0));
59+
60+
auto c1 = c.next();
61+
REQUIRE(c1.has_value());
62+
CHECK(c1->get_int() == static_cast<int64_t>(r * 100 + 1));
63+
64+
auto c2 = c1->next();
65+
REQUIRE(c2.has_value());
66+
CHECK(c2->get_int() == static_cast<int64_t>(r * 100 + 2));
67+
}
68+
69+
// First and last.
70+
CHECK(idx.front().get_int() == 0);
71+
CHECK(idx.back().get_int()
72+
== static_cast<int64_t>((NROWS - 1) * 100 + 0));
73+
}
74+
75+
TEST_CASE("sparse_cursor_container-empty")
76+
{
77+
lnav::sparse_cursor_container<> idx;
78+
CHECK(idx.empty());
79+
CHECK(idx.size() == 0);
80+
CHECK(idx.block_count() == 0);
81+
}
82+
83+
TEST_CASE("sparse_cursor_container-single-row")
84+
{
85+
lnav::cell_container cells;
86+
lnav::sparse_cursor_container<16> idx;
87+
idx.set_columns(2);
88+
89+
idx.push_row(cells.end_cursor());
90+
cells.push_text_cell(string_fragment::from_const("hello"));
91+
cells.push_int_cell(42);
92+
93+
CHECK(idx.size() == 1);
94+
CHECK(idx.block_count() == 1);
95+
96+
auto c = idx.at(0);
97+
CHECK(c.get_type() == lnav::cell_type::CT_TEXT);
98+
CHECK(c.get_text() == "hello");
99+
}
100+
101+
TEST_CASE("sparse_cursor_container-spans-chunk-boundaries")
102+
{
103+
// A `cell_container` chunk is ~32 KiB; each int cell is 9 bytes,
104+
// so two cells per row is 18 bytes/row. Pushing 10000 rows forces
105+
// several new-chunk allocations, which is the case where a block-
106+
// start cursor captured by `end_cursor()` ends up pinned to a
107+
// chunk whose capacity was then exhausted — the cursor's
108+
// `c_offset` equals the old chunk's `cc_size`, and a subsequent
109+
// `next()` call reads past valid data unless `at()` syncs first.
110+
constexpr size_t NCOLS = 2;
111+
constexpr size_t NROWS = 10000;
112+
113+
lnav::cell_container cells;
114+
lnav::sparse_cursor_container<64> idx;
115+
idx.set_columns(NCOLS);
116+
117+
for (size_t r = 0; r < NROWS; ++r) {
118+
idx.push_row(cells.end_cursor());
119+
cells.push_int_cell(static_cast<int64_t>(r));
120+
cells.push_int_cell(static_cast<int64_t>(r + 1000000));
121+
}
122+
123+
CHECK(idx.size() == NROWS);
124+
// Every row's first and second cell should read back its seeded
125+
// value, no matter where in the chunk chain it sits.
126+
for (size_t r = 0; r < NROWS; ++r) {
127+
auto c0 = idx.at(r);
128+
CHECK_MESSAGE(c0.get_type() == lnav::cell_type::CT_INTEGER,
129+
"row ", r);
130+
CHECK_MESSAGE(c0.get_int() == static_cast<int64_t>(r),
131+
"row ", r);
132+
133+
auto c1 = c0.next();
134+
REQUIRE_MESSAGE(c1.has_value(), "row ", r);
135+
CHECK_MESSAGE(c1->get_int() == static_cast<int64_t>(r + 1000000),
136+
"row ", r);
137+
}
138+
}
139+
140+
TEST_CASE("sparse_cursor_container-clear")
141+
{
142+
lnav::cell_container cells;
143+
lnav::sparse_cursor_container<16> idx;
144+
idx.set_columns(1);
145+
146+
for (int i = 0; i < 50; ++i) {
147+
idx.push_row(cells.end_cursor());
148+
cells.push_int_cell(i);
149+
}
150+
CHECK(idx.size() == 50);
151+
152+
idx.clear();
153+
cells.reset();
154+
155+
CHECK(idx.empty());
156+
CHECK(idx.block_count() == 0);
157+
158+
// Still usable after clear.
159+
idx.push_row(cells.end_cursor());
160+
cells.push_int_cell(999);
161+
CHECK(idx.size() == 1);
162+
CHECK(idx.at(0).get_int() == 999);
163+
}

‎src/cmds.io.cc‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -507,7 +507,7 @@ com_save_to(exec_context& ec,
507507
break;
508508
}
509509

510-
auto& row_cursor = dls.dls_row_cursors[row];
510+
auto row_cursor = dls.dls_row_cursors[row];
511511
first = true;
512512
auto cursor = row_cursor.sync();
513513
for (size_t lpc = 0; lpc < dls.dls_headers.size();
@@ -981,7 +981,7 @@ com_save_to(exec_context& ec,
981981
break;
982982
}
983983

984-
const auto& row_cursor = dls.dls_row_cursors[row];
984+
const auto row_cursor = dls.dls_row_cursors[row];
985985
auto cursor = row_cursor.sync();
986986
for (size_t lpc = 0; lpc < dls.dls_headers.size();
987987
lpc++, cursor = cursor->next())

‎src/command_executor.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1063,7 +1063,7 @@ sql_callback(exec_context& ec, sqlite3_stmt* stmt)
10631063
}
10641064
}
10651065

1066-
dls.dls_row_cursors.emplace_back(dls.dls_cell_container.end_cursor());
1066+
dls.dls_row_cursors.push_row(dls.dls_cell_container.end_cursor());
10671067
dls.dls_push_column = 0;
10681068
for (int lpc = 0; lpc < ncols; lpc++) {
10691069
const auto value_type = sqlite3_column_type(stmt, lpc);

0 commit comments

Comments
 (0)