diff --git a/convex-core/src/main/java/convex/core/store/ACachedStore.java b/convex-core/src/main/java/convex/core/store/ACachedStore.java index a0103ee32..d2bbc1bbb 100644 --- a/convex-core/src/main/java/convex/core/store/ACachedStore.java +++ b/convex-core/src/main/java/convex/core/store/ACachedStore.java @@ -13,7 +13,7 @@ */ public abstract class ACachedStore extends AStore { - protected final RefCache refCache=RefCache.create(10000); + protected final RefCache refCache=RefCache.create(Integer.getInteger("convex.cache.size", 10000)); /** * Store-bound encoder. Manages thread-local store context during decode. diff --git a/convex-core/src/main/java/convex/etch/Etch.java b/convex-core/src/main/java/convex/etch/Etch.java index 2d87bba30..2e209cfd0 100644 --- a/convex-core/src/main/java/convex/etch/Etch.java +++ b/convex-core/src/main/java/convex/etch/Etch.java @@ -292,14 +292,9 @@ private synchronized MappedByteBuffer createBuffer(int regionIndex) throws IOExc long pos=((long)regionIndex)*MAX_REGION_SIZE; // Expand region size until big enough for current database plus appropriate margin - int length; - if (regionIndex==0) { - length=1<<16; - while((length Ref setRootData(T data) throws IOException { return ref; } + /** + * Compacts this store into a new Etch file containing only the cells reachable + * from the current root. Dead cells accumulated through updates are discarded. + * + *

The source store must have its root data set (via {@link #setRootData}). + * The target file must not exist or be empty. + * + *

Compaction ratio depends on write history: a store that has had many updates + * to the same rows may see 10× or more reduction because every Index node touched + * during an insert is written as a new cell in the append-only file. + * + *

Thread safety: the caller is responsible for ensuring no concurrent writes + * to this store during compaction. + * + * @param targetFile Destination file for the compacted store (must be writable) + * @return New EtchStore backed by the compacted file + * @throws IOException If an IO error occurs during read or write + */ + public EtchStore compact(File targetFile) throws IOException { + Ref rootRef = getRootRef(); + if (rootRef == null) throw new IOException("Source store has no root data — call setRootData first"); + // Load the full reachable cell tree into memory/cache in the source store context + rootRef = storeTopRef(rootRef, Ref.PERSISTED, null); + EtchStore target = EtchStore.create(targetFile); + target.setRootData(rootRef.getValue()); + return target; + } + /** * Gets the underlying Etch instance - * + * * @return Etch instance */ public Etch getEtch() { diff --git a/convex-db/pom.xml b/convex-db/pom.xml index fff13a4bb..e5fdf69ac 100644 --- a/convex-db/pom.xml +++ b/convex-db/pom.xml @@ -90,5 +90,13 @@ 42.7.7 test + + + + org.mariadb.jdbc + mariadb-java-client + 3.4.1 + test + diff --git a/convex-db/src/main/java/convex/db/calcite/ConvexSchema.java b/convex-db/src/main/java/convex/db/calcite/ConvexSchema.java index f61408224..cd52f7d59 100644 --- a/convex-db/src/main/java/convex/db/calcite/ConvexSchema.java +++ b/convex-db/src/main/java/convex/db/calcite/ConvexSchema.java @@ -2,6 +2,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import org.apache.calcite.schema.Schema; import org.apache.calcite.schema.Table; @@ -27,6 +28,12 @@ public class ConvexSchema extends AbstractSchema { private final SQLSchema tables; private final String name; + /** + * Registry of SQL index names to (tableName, columnName) pairs. + * Populated by CREATE INDEX DDL; used to resolve DROP INDEX by name. + */ + final ConcurrentHashMap indexRegistry = new ConcurrentHashMap<>(); + /** * Creates a new ConvexSchema backed by the given database. * @@ -130,4 +137,43 @@ public boolean tableExists(String tableName) { public ConvexTable getConvexTable(String tableName) { return new ConvexTable(this, tableName); } + + /** + * Creates a secondary index via DDL. + * + * @param indexName SQL index name (for later DROP INDEX lookup) + * @param tableName Table to index + * @param columnName Column to index + * @param ifNotExists if true, silently succeed if already exists + * @return true if index was created + */ + public boolean createIndex(String indexName, String tableName, String columnName, + boolean ifNotExists) { + if (tables.hasIndex(tableName, columnName)) { + if (ifNotExists) return false; + throw new IllegalStateException( + "Index on " + tableName + "(" + columnName + ") already exists"); + } + boolean created = tables.createIndex(tableName, columnName); + if (created) { + indexRegistry.put(indexName.toUpperCase(), new String[]{tableName, columnName}); + } + return created; + } + + /** + * Drops a secondary index by its SQL name. + * + * @param indexName SQL index name + * @param ifExists if true, silently succeed if not found + * @return true if index was dropped + */ + public boolean dropIndex(String indexName, boolean ifExists) { + String[] pair = indexRegistry.remove(indexName.toUpperCase()); + if (pair == null) { + if (ifExists) return false; + throw new IllegalStateException("Index not found: " + indexName); + } + return tables.dropIndex(pair[0], pair[1]); + } } diff --git a/convex-db/src/main/java/convex/db/calcite/ConvexTable.java b/convex-db/src/main/java/convex/db/calcite/ConvexTable.java index f98198ec5..07ea1b953 100644 --- a/convex-db/src/main/java/convex/db/calcite/ConvexTable.java +++ b/convex-db/src/main/java/convex/db/calcite/ConvexTable.java @@ -47,6 +47,12 @@ public class ConvexTable extends AbstractQueryableTable private final ConvexSchema schema; private final String tableName; + // Lazily cached column metadata — column names and types are stable for a + // given table definition and expensive to re-derive from the lattice on every + // getRowType() / getStatistic() call during query planning. + private volatile String[] cachedColumnNames; + private volatile ConvexColumnType[] cachedColumnTypes; + public ConvexTable(ConvexSchema schema, String tableName) { super(Object[].class); this.schema = schema; @@ -57,9 +63,8 @@ public ConvexTable(ConvexSchema schema, String tableName) { public RelDataType getRowType(RelDataTypeFactory typeFactory) { RelDataTypeFactory.Builder builder = typeFactory.builder(); - SQLSchema tables = schema.getTables(); - String[] columnNames = tables.getColumnNames(tableName); - ConvexColumnType[] columnTypes = tables.getColumnTypes(tableName); + String[] columnNames = getColumnNames(); + ConvexColumnType[] columnTypes = getColumnTypes(); if (columnNames != null && columnTypes != null) { for (int i = 0; i < columnNames.length; i++) { @@ -71,6 +76,13 @@ public RelDataType getRowType(RelDataTypeFactory typeFactory) { return builder.build(); } + private String[] getColumnNames() { + if (cachedColumnNames == null) { + cachedColumnNames = schema.getTables().getColumnNames(tableName); + } + return cachedColumnNames; + } + /** * Returns a ConvexTableScan in CONVEX convention. This is Calcite's * standard extension point for custom adapters — bypasses EnumerableTableScan @@ -87,7 +99,10 @@ public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable relOptTable) } public ConvexColumnType[] getColumnTypes() { - return schema.getTables().getColumnTypes(tableName); + if (cachedColumnTypes == null) { + cachedColumnTypes = schema.getTables().getColumnTypes(tableName); + } + return cachedColumnTypes; } /** diff --git a/convex-db/src/main/java/convex/db/calcite/ConvexTableEnumerator.java b/convex-db/src/main/java/convex/db/calcite/ConvexTableEnumerator.java index 5ee4374d8..b9ae111cf 100644 --- a/convex-db/src/main/java/convex/db/calcite/ConvexTableEnumerator.java +++ b/convex-db/src/main/java/convex/db/calcite/ConvexTableEnumerator.java @@ -1,64 +1,122 @@ package convex.db.calcite; -import java.util.ArrayList; -import java.util.List; +import java.util.Iterator; import convex.core.data.ABlob; import convex.core.data.ACell; import convex.core.data.AVector; import convex.core.data.Index; +import convex.db.lattice.RowBlock; import convex.db.lattice.SQLRow; import convex.db.lattice.SQLSchema; import convex.db.lattice.SQLTable; /** - * Enumerator that performs a full table scan from a lattice Index. + * Enumerator that performs a lazy full table scan over a block-packed row index. * - *

Rows are returned in Index key order (sorted by PK). - * Pre-collects via forEach for O(n) traversal instead of O(n*depth) entryAt(). + *

Iterates the outer block Index one block at a time, decoding each block's + * bytes once into a row buffer, then draining that buffer row by row. + * + *

Two traversal paths: + *

*/ public class ConvexTableEnumerator extends ConvexEnumerator { - private final List rows; - private int position = -1; + /** Non-null when blockVec path is active; stored to allow reset(). */ + private final AVector blockVec; + /** O(n)-total iterator over blockVec; reset()-able by re-creating from blockVec. */ + private Iterator blockVecIter; + /** Fallback Index path (used when blockVec == null). */ + private final Index blockIndex; + private final int colCount; + private final long blockCount; + + private long blockPos = -1; + @SuppressWarnings("unchecked") + private AVector[] blockRowBuf = new AVector[0]; + private int blockRowBufSize = 0; + private int rowPos = 0; - /** - * Creates an enumerator for a full table scan. - */ public ConvexTableEnumerator(SQLSchema tables, String tableName) { SQLTable table = tables.getLiveTable(tableName); if (table == null) { - this.rows = List.of(); + this.blockVec = null; + this.blockVecIter = null; + this.blockIndex = Index.none(); + this.colCount = 0; + this.blockCount = 0; return; } - Index> rawRows = table.getRows(); - if (rawRows == null) { - this.rows = List.of(); - return; + AVector bv = table.getBlockVec(); + if (bv != null) { + // blockVec path: iterator is O(n) total (visits each VectorTree node once) + this.blockVec = bv; + this.blockVecIter = bv.iterator(); + this.blockIndex = null; + this.blockCount = bv.count(); + } else { + this.blockVec = null; + this.blockVecIter = null; + Index rows = table.getRows(); + this.blockIndex = rows != null ? rows : Index.none(); + this.blockCount = this.blockIndex.count(); } - // Single-pass tree traversal, skip tombstones, unwrap values - List collected = new ArrayList<>((int) Math.min(rawRows.count(), Integer.MAX_VALUE)); - rawRows.forEach((k, v) -> { - if (SQLRow.isLive(v)) { - collected.add(SQLRow.getValues(v).toCellArray()); - } - }); - this.rows = collected; + AVector> schema = table.getSchema(); + this.colCount = schema != null ? (int) schema.count() : 0; } @Override public boolean moveNext() { - if (++position < rows.size()) { - currentRow = rows.get(position); - return true; + while (true) { + // Drain rows buffered from the current block + while (rowPos < blockRowBufSize) { + AVector row = blockRowBuf[rowPos++]; + if (!SQLRow.isLive(row)) continue; + ACell[] arr = SQLRow.getValues(row).toCellArray(); + if (arr.length < colCount) { + ACell[] padded = new ACell[colCount]; + System.arraycopy(arr, 0, padded, 0, arr.length); + arr = padded; + } + currentRow = arr; + return true; + } + // Advance to next block — decode it once into blockRowBuf + blockPos++; + if (blockPos >= blockCount) { + currentRow = null; + return false; + } + ACell block = (blockVecIter != null) + ? blockVecIter.next() + : blockIndex.entryAt(blockPos).getValue(); + int n = RowBlock.isBlock(block) ? RowBlock.count(block) : 0; + if (n > blockRowBuf.length) { + @SuppressWarnings("unchecked") + AVector[] newBuf = new AVector[n]; + blockRowBuf = newBuf; + } + blockRowBufSize = 0; + rowPos = 0; + final int[] idx = {0}; + RowBlock.forEach(block, (pk, row) -> blockRowBuf[idx[0]++] = row); + blockRowBufSize = idx[0]; } - currentRow = null; - return false; } @Override public void reset() { - position = -1; + blockPos = -1; + blockRowBufSize = 0; + rowPos = 0; currentRow = null; + if (blockVec != null) blockVecIter = blockVec.iterator(); } } diff --git a/convex-db/src/main/java/convex/db/calcite/rel/ConvexResultConverter.java b/convex-db/src/main/java/convex/db/calcite/rel/ConvexResultConverter.java index f4ea5fec9..2859a2fa9 100644 --- a/convex-db/src/main/java/convex/db/calcite/rel/ConvexResultConverter.java +++ b/convex-db/src/main/java/convex/db/calcite/rel/ConvexResultConverter.java @@ -1,7 +1,6 @@ package convex.db.calcite.rel; -import java.util.ArrayList; -import java.util.List; +import java.util.Iterator; import org.apache.calcite.DataContext; import org.apache.calcite.linq4j.Enumerable; @@ -30,33 +29,47 @@ public class ConvexResultConverter { /** * Executes in ARRAY format — each row is {@code Object[]}. * Used for multi-column results. + * + *

Returns a lazy {@code Enumerable}: no rows are decoded until Calcite + * iterates via {@code rs.next()}, keeping the heap footprint near-zero + * at query-open time. */ public static Enumerable execute(ConvexRel rel, int fieldCount, DataContext ctx) { ConvexEnumerable convexResult = rel.execute(ctx); - - List results = new ArrayList<>(); - for (ACell[] row : convexResult) { - Object[] javaRow = new Object[fieldCount]; - for (int i = 0; i < Math.min(row.length, fieldCount); i++) { - javaRow[i] = cellToJava(row[i]); - } - results.add(javaRow); - } - return Linq4j.asEnumerable(results); + return Linq4j.asEnumerable(() -> { + Iterator it = convexResult.iterator(); + return new Iterator() { + @Override public boolean hasNext() { return it.hasNext(); } + @Override public Object[] next() { + ACell[] row = it.next(); + Object[] javaRow = new Object[fieldCount]; + for (int i = 0; i < Math.min(row.length, fieldCount); i++) { + javaRow[i] = cellToJava(row[i]); + } + return javaRow; + } + }; + }); } /** * Executes in SCALAR format — each element is the column value directly. * Used for single-column results. + * + *

Returns a lazy {@code Enumerable}: no rows are decoded until iteration. */ public static Enumerable executeScalar(ConvexRel rel, DataContext ctx) { ConvexEnumerable convexResult = rel.execute(ctx); - - List results = new ArrayList<>(); - for (ACell[] row : convexResult) { - results.add(row.length > 0 ? cellToJava(row[0]) : null); - } - return Linq4j.asEnumerable(results); + return Linq4j.asEnumerable(() -> { + Iterator it = convexResult.iterator(); + return new Iterator() { + @Override public boolean hasNext() { return it.hasNext(); } + @Override public Object next() { + ACell[] row = it.next(); + return row.length > 0 ? cellToJava(row[0]) : null; + } + }; + }); } /** diff --git a/convex-db/src/main/java/convex/db/calcite/rel/ConvexSort.java b/convex-db/src/main/java/convex/db/calcite/rel/ConvexSort.java index bd5b7d78b..9a12736aa 100644 --- a/convex-db/src/main/java/convex/db/calcite/rel/ConvexSort.java +++ b/convex-db/src/main/java/convex/db/calcite/rel/ConvexSort.java @@ -16,21 +16,34 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.DataContext; +import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.type.SqlTypeName; +import convex.core.data.ABlob; import convex.core.data.ACell; +import convex.core.data.AVector; +import convex.core.data.Index; import convex.core.lang.RT; +import convex.db.calcite.ConvexSchema; +import convex.db.calcite.ConvexTable; import convex.db.calcite.convention.ConvexConvention; import convex.db.calcite.convention.ConvexEnumerable; import convex.db.calcite.convention.ConvexRel; import convex.db.calcite.eval.ConvexExpressionEvaluator; +import convex.db.lattice.RowBlock; +import convex.db.lattice.SQLRow; +import convex.db.lattice.SQLTable; /** * Sort in CONVEX convention. * *

Sorts ACell[] rows using type-specific comparators selected from * the column's SQL type at plan time. + * + *

For {@code ORDER BY PK [ASC|DESC] LIMIT N} directly over a table scan, + * uses index-native traversal via {@code entryAt()} to avoid a full scan and + * in-memory sort. Only N index nodes are visited instead of all n rows. */ public class ConvexSort extends Sort implements ConvexRel { @@ -56,6 +69,13 @@ public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { @Override public ConvexEnumerable execute(DataContext ctx) { + // Index-native path: ORDER BY PK [ASC|DESC] LIMIT N with no OFFSET, + // directly over a table scan (no intervening filter or project). + if (fetch != null && offset == null && getInput() instanceof ConvexTableScan scan) { + ConvexEnumerable pushed = tryIndexScan(scan, ctx); + if (pushed != null) return pushed; + } + ConvexRel inputRel = (ConvexRel) getInput(); ConvexEnumerable input = inputRel.execute(ctx); @@ -103,6 +123,81 @@ public ConvexEnumerable execute(DataContext ctx) { return ConvexEnumerable.of(rows); } + /** + * Tries to serve {@code ORDER BY PK [ASC|DESC] LIMIT N} directly from the index, + * walking forward or backward with {@code entryAt()} and stopping as soon as + * {@code fetchVal} live rows have been collected. + * + *

Applicable when: + *

    + *
  • collation is a single field on column 0 (the primary key)
    • + *
  • LIMIT is a literal (no OFFSET)
    • + *
  • input is a bare {@link ConvexTableScan} (no filter, no project)
    • + *
+ * + * @return a {@link ConvexEnumerable} with at most {@code fetchVal} rows, or + * {@code null} if the optimisation is not applicable + */ + private ConvexEnumerable tryIndexScan(ConvexTableScan scan, DataContext ctx) { + // Single-field sort on PK (column 0) only + List fields = collation.getFieldCollations(); + if (fields.size() != 1 || fields.get(0).getFieldIndex() != 0) return null; + + // LIMIT must be a literal + if (!(fetch instanceof RexLiteral)) return null; + ACell fetchCell = ConvexExpressionEvaluator.literalToCell((RexLiteral) fetch); + if (fetchCell == null) return null; + int fetchVal = ((Number) RT.jvm(fetchCell)).intValue(); + if (fetchVal <= 0) return ConvexEnumerable.empty(); + + // Locate the raw index + ConvexTable convexTable = scan.getTable().unwrap(ConvexTable.class); + if (convexTable == null) return null; + ConvexSchema schema = convexTable.getSchema(); + SQLTable sqlTable = schema.getTables().getLiveTable(convexTable.getTableName()); + if (sqlTable == null) return ConvexEnumerable.empty(); + Index rawRows = sqlTable.getRows(); + if (rawRows == null) return ConvexEnumerable.empty(); + + long total = rawRows.count(); + if (total == 0) return ConvexEnumerable.empty(); + + boolean desc = fields.get(0).getDirection().isDescending(); + List result = new ArrayList<>(Math.min(fetchVal, (int) Math.min(total, Integer.MAX_VALUE))); + + if (desc) { + // Walk backwards through blocks, then rows within each block + for (long i = total - 1; i >= 0 && result.size() < fetchVal; i--) { + ACell entry = rawRows.entryAt(i).getValue(); + if (RowBlock.isBlock(entry)) { + int blockSize = RowBlock.count(entry); + for (int j = blockSize - 1; j >= 0 && result.size() < fetchVal; j--) { + AVector row = RowBlock.getRow(entry, j); + if (SQLRow.isLive(row)) result.add(SQLRow.getValues(row).toCellArray()); + } + } else if (SQLRow.isLive((AVector)entry)) { + result.add(SQLRow.getValues((AVector)entry).toCellArray()); + } + } + } else { + // Walk forwards through blocks, then rows within each block + for (long i = 0; i < total && result.size() < fetchVal; i++) { + ACell entry = rawRows.entryAt(i).getValue(); + if (RowBlock.isBlock(entry)) { + int blockSize = RowBlock.count(entry); + for (int j = 0; j < blockSize && result.size() < fetchVal; j++) { + AVector row = RowBlock.getRow(entry, j); + if (SQLRow.isLive(row)) result.add(SQLRow.getValues(row).toCellArray()); + } + } else if (SQLRow.isLive((AVector)entry)) { + result.add(SQLRow.getValues((AVector)entry).toCellArray()); + } + } + } + + return ConvexEnumerable.of(result); + } + /** * Creates a row comparator by selecting a type-specific cell comparator * for each sort field based on the column's SQL type. diff --git a/convex-db/src/main/java/convex/db/calcite/rel/ConvexTableScan.java b/convex-db/src/main/java/convex/db/calcite/rel/ConvexTableScan.java index 7fd0d1050..35cbee2c0 100644 --- a/convex-db/src/main/java/convex/db/calcite/rel/ConvexTableScan.java +++ b/convex-db/src/main/java/convex/db/calcite/rel/ConvexTableScan.java @@ -1,6 +1,5 @@ package convex.db.calcite.rel; -import java.util.ArrayList; import java.util.List; import org.apache.calcite.DataContext; @@ -12,18 +11,13 @@ import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.metadata.RelMetadataQuery; -import convex.core.data.ABlob; import convex.core.data.ACell; -import convex.core.data.AVector; -import convex.core.data.Index; import convex.db.calcite.ConvexSchema; import convex.db.calcite.ConvexTable; import convex.db.calcite.convention.ConvexConvention; import convex.db.calcite.convention.ConvexEnumerable; import convex.db.calcite.convention.ConvexRel; -import convex.db.lattice.SQLRow; import convex.db.lattice.SQLSchema; -import convex.db.lattice.SQLTable; /** * Table scan in CONVEX convention. @@ -56,20 +50,34 @@ public ConvexEnumerable execute(DataContext ctx) { SQLSchema tables = schema.getTables(); String tableName = convexTable.getTableName(); - SQLTable sqlTable = tables.getLiveTable(tableName); - if (sqlTable == null) return ConvexEnumerable.empty(); + // Lazy streaming scan: no full materialisation of ACell[] rows. + // ConvexTableEnumerator decodes one block at a time, avoiding a + // full-result-set heap spike at query-open time. + return () -> { + convex.db.calcite.ConvexTableEnumerator e = + new convex.db.calcite.ConvexTableEnumerator(tables, tableName); + return new java.util.Iterator() { + private boolean primed = false; + private boolean hasNext = false; - Index> rawRows = sqlTable.getRows(); - if (rawRows == null) return ConvexEnumerable.empty(); + private void prime() { + if (!primed) { + hasNext = e.moveNext(); + primed = true; + } + } - // Single-pass tree traversal via forEach, skip tombstones, unwrap values - List result = new ArrayList<>((int) Math.min(rawRows.count(), Integer.MAX_VALUE)); - rawRows.forEach((k, v) -> { - if (SQLRow.isLive(v)) { - result.add(SQLRow.getValues(v).toCellArray()); - } - }); + @Override public boolean hasNext() { prime(); return hasNext; } - return ConvexEnumerable.of(result); + @Override public ACell[] next() { + prime(); + if (!hasNext) throw new java.util.NoSuchElementException(); + ACell[] row = e.current(); + hasNext = e.moveNext(); + primed = true; + return row; + } + }; + }; } } diff --git a/convex-db/src/main/java/convex/db/lattice/BlockTableLattice.java b/convex-db/src/main/java/convex/db/lattice/BlockTableLattice.java new file mode 100644 index 000000000..afff07d7c --- /dev/null +++ b/convex-db/src/main/java/convex/db/lattice/BlockTableLattice.java @@ -0,0 +1,47 @@ +package convex.db.lattice; + +import convex.core.data.ABlob; +import convex.core.data.ACell; +import convex.core.data.Index; +import convex.lattice.ALattice; +import convex.lattice.generic.IndexLattice; + +/** + * Lattice for the block-packed row index of a SQL table. + * + *

Replaces {@link TableLattice} in the lattice path. The value is an + * {@code Index} keyed by block prefix, where each value + * is a {@link RowBlock}-encoded flat Blob rather than a single row entry. + * Per-block merge is handled by {@link RowBlockLattice}. + */ +public class BlockTableLattice extends ALattice> { + + public static final BlockTableLattice INSTANCE = new BlockTableLattice(); + + private final IndexLattice delegate = + IndexLattice.create(RowBlockLattice.INSTANCE); + + private BlockTableLattice() {} + + @Override + @SuppressWarnings("unchecked") + public Index zero() { + return (Index) Index.EMPTY; + } + + @Override + public Index merge(Index a, Index b) { + return delegate.merge(a, b); + } + + @Override + public boolean checkForeign(Index value) { + return delegate.checkForeign(value); + } + + @SuppressWarnings("unchecked") + @Override + public ALattice path(ACell childKey) { + return delegate.path(childKey); + } +} diff --git a/convex-db/src/main/java/convex/db/lattice/ColumnIndex.java b/convex-db/src/main/java/convex/db/lattice/ColumnIndex.java new file mode 100644 index 000000000..0cfa78483 --- /dev/null +++ b/convex-db/src/main/java/convex/db/lattice/ColumnIndex.java @@ -0,0 +1,149 @@ +package convex.db.lattice; + +import java.nio.charset.StandardCharsets; + +import convex.core.data.ABlob; +import convex.core.data.ACell; +import convex.core.data.AString; +import convex.core.data.Blob; +import convex.core.data.prim.CVMLong; + +/** + * Utility class for secondary column index key encoding and matching. + * + *

Index key format: {@code [valueLenHi, valueLenLo, value_bytes..., pk_bytes...]} + *

    + *
  • Bytes 0–1: big-endian unsigned length of the encoded value (max 65535 bytes)
    • + *
  • Bytes 2 to 2+valueLen: sortable encoding of the column value
    • + *
  • Remaining bytes: primary key bytes (as produced by {@link SQLSchema#toKey})
    • + *
+ * + *

The value encoding is sortable, enabling range queries: + *

    + *
  • {@link CVMLong}: 8 bytes big-endian with sign bit XOR'd for correct signed ordering
    • + *
  • {@link AString}: raw UTF-8 bytes (lexicographic order)
    • + *
  • Other: CAD3 cell encoding (exact match only)
    • + *
+ */ +public class ColumnIndex { + + private ColumnIndex() {} + + // ── Value encoding ─────────────────────────────────────────────────────── + + /** + * Encodes a column value to a sortable byte array for use as index key prefix. + */ + public static byte[] encodeValue(ACell value) { + if (value instanceof CVMLong n) { + // 8-byte big-endian with sign bit flipped for signed ordering + long v = n.longValue() ^ Long.MIN_VALUE; + byte[] b = new byte[8]; + for (int i = 7; i >= 0; i--) { b[i] = (byte) (v & 0xFF); v >>= 8; } + return b; + } + if (value instanceof AString s) { + return s.toString().getBytes(StandardCharsets.UTF_8); + } + // Fallback: CAD3 encoding (exact match only, not range-sortable across types) + Blob encoded = convex.core.data.Cells.encode(value); + byte[] b = new byte[(int) encoded.count()]; + for (int i = 0; i < b.length; i++) b[i] = encoded.byteAtUnchecked(i); + return b; + } + + // ── Key construction ───────────────────────────────────────────────────── + + /** + * Builds the index key for a (value, pk) pair. + * Format: 2-byte-big-endian(valueLen) ++ value_bytes ++ pk_bytes + */ + public static ABlob indexKey(ACell value, ABlob pk) { + byte[] vb = encodeValue(value); + byte[] pkb = blobBytes(pk); + byte[] key = new byte[2 + vb.length + pkb.length]; + key[0] = (byte) (vb.length >> 8); + key[1] = (byte) (vb.length); + System.arraycopy(vb, 0, key, 2, vb.length); + System.arraycopy(pkb, 0, key, 2 + vb.length, pkb.length); + return Blob.wrap(key); + } + + // ── Key inspection ─────────────────────────────────────────────────────── + + /** + * Returns the stored value length (bytes 0–1 of the index key). + */ + public static int getValueLen(ABlob indexKey) { + return ((indexKey.byteAtUnchecked(0) & 0xFF) << 8) + | (indexKey.byteAtUnchecked(1) & 0xFF); + } + + /** + * Extracts the primary key blob from an index key. + */ + public static ABlob extractPk(ABlob indexKey) { + int valueLen = getValueLen(indexKey); + int pkStart = 2 + valueLen; + int pkLen = (int) indexKey.count() - pkStart; + if (pkLen <= 0) return Blob.EMPTY; + byte[] pkb = new byte[pkLen]; + for (int i = 0; i < pkLen; i++) pkb[i] = indexKey.byteAtUnchecked(pkStart + i); + return Blob.wrap(pkb); + } + + // ── Match predicates ───────────────────────────────────────────────────── + + /** + * Returns true if the index key encodes exactly {@code value}. + */ + public static boolean matchesValue(ABlob indexKey, ACell value) { + byte[] vb = encodeValue(value); + if (indexKey.count() < 2 + vb.length) return false; + int storedLen = getValueLen(indexKey); + if (storedLen != vb.length) return false; + for (int i = 0; i < vb.length; i++) { + if (indexKey.byteAtUnchecked(2 + i) != vb[i]) return false; + } + return true; + } + + /** + * Returns true if the value encoded in the index key is in the inclusive range [from, to]. + * + *

Requires that {@code from} and {@code to} produce same-length encodings + * (true for CVMLong, true for fixed-length value types). If lengths differ, + * returns false. + */ + public static boolean matchesRange(ABlob indexKey, ACell from, ACell to) { + byte[] fromBytes = encodeValue(from); + byte[] toBytes = encodeValue(to); + if (fromBytes.length != toBytes.length) return false; + int storedLen = getValueLen(indexKey); + if (storedLen != fromBytes.length) return false; + + // Lexicographic comparison of stored value bytes vs [from, to] + int cmpFrom = 0, cmpTo = 0; + for (int i = 0; i < fromBytes.length; i++) { + int b = indexKey.byteAtUnchecked(2 + i) & 0xFF; + if (cmpFrom == 0) { + if (b > (fromBytes[i] & 0xFF)) cmpFrom = 1; + else if (b < (fromBytes[i] & 0xFF)) cmpFrom = -1; + } + if (cmpTo == 0) { + if (b < (toBytes[i] & 0xFF)) cmpTo = -1; + else if (b > (toBytes[i] & 0xFF)) cmpTo = 1; + } + } + return cmpFrom >= 0 && cmpTo <= 0; // key_val >= from AND key_val <= to + } + + // ── Internal helpers ───────────────────────────────────────────────────── + + static byte[] blobBytes(ABlob blob) { + int n = (int) blob.count(); + byte[] b = new byte[n]; + for (int i = 0; i < n; i++) b[i] = blob.byteAtUnchecked(i); + return b; + } +} diff --git a/convex-db/src/main/java/convex/db/lattice/HistoryKey.java b/convex-db/src/main/java/convex/db/lattice/HistoryKey.java new file mode 100644 index 000000000..ce69ecaa4 --- /dev/null +++ b/convex-db/src/main/java/convex/db/lattice/HistoryKey.java @@ -0,0 +1,90 @@ +package convex.db.lattice; + +import convex.core.data.ABlob; +import convex.core.data.Blob; + +/** + * Encoding and decoding utilities for history index keys. + * + *

Key format: {@code [1 byte: pkLength] [pkLength bytes: pk] [8 bytes: nanotime big-endian]} + * + *

The 1-byte length prefix supports pk blobs up to 255 bytes, covering all practical + * key types (CVMLong = 8 bytes, typical strings well within 255). + * + *

Big-endian nanotime ensures that entries for the same pk are sorted chronologically + * in the radix tree — {@code forEach} yields them oldest-first with no extra sorting. + */ +public class HistoryKey { + + static final int NANOTIME_BYTES = 8; + static final int LENGTH_BYTES = 1; + + private HistoryKey() {} + + /** + * Creates a history index key from a pk blob and a nanotime. + * + * @param pk Primary key blob (max 255 bytes) + * @param nanotime Monotonic timestamp from {@link System#nanoTime()} + * @return History key blob + */ + public static ABlob of(ABlob pk, long nanotime) { + int pkLen = (int) pk.count(); + if (pkLen > 255) throw new IllegalArgumentException("Primary key too long: " + pkLen); + byte[] key = new byte[LENGTH_BYTES + pkLen + NANOTIME_BYTES]; + key[0] = (byte) pkLen; + for (int i = 0; i < pkLen; i++) key[LENGTH_BYTES + i] = byteAt(pk, i); + long v = nanotime; + for (int i = NANOTIME_BYTES - 1; i >= 0; i--) { key[LENGTH_BYTES + pkLen + i] = (byte)(v & 0xFF); v >>>= 8; } + return Blob.wrap(key); + } + + /** + * Returns the prefix blob used to scan all history entries for a given pk. + * This is the {@code [pkLength | pk bytes]} portion without the nanotime suffix. + * + * @param pk Primary key blob + * @return Prefix blob for radix-tree iteration + */ + public static ABlob prefix(ABlob pk) { + int pkLen = (int) pk.count(); + byte[] prefix = new byte[LENGTH_BYTES + pkLen]; + prefix[0] = (byte) pkLen; + for (int i = 0; i < pkLen; i++) prefix[LENGTH_BYTES + i] = byteAt(pk, i); + return Blob.wrap(prefix); + } + + /** + * Returns true if {@code hkey} starts with all nibbles of {@code pkPrefix}. + * Used to filter history entries belonging to a specific pk during iteration. + */ + public static boolean hasPrefix(ABlob hkey, ABlob pkPrefix) { + long pNibbles = pkPrefix.hexLength(); + if (hkey.hexLength() < pNibbles) return false; + for (long i = 0; i < pNibbles; i++) { + if (hkey.getHexDigit(i) != pkPrefix.getHexDigit(i)) return false; + } + return true; + } + + /** + * Extracts the nanotime from a history key. + * + * @param hkey History key blob + * @return Nanotime value embedded in the key + */ + public static long extractNanotime(ABlob hkey) { + // First byte is pkLength; nanotime starts at byte (1 + pkLength) + int pkLen = (hkey.getHexDigit(0) << 4) | hkey.getHexDigit(1); + long ts = 0; + int startNibble = 2 + 2 * pkLen; // skip [length byte] + [pk bytes] + for (int i = 0; i < 16; i++) ts = (ts << 4) | hkey.getHexDigit(startNibble + i); + return ts; + } + + /** Reads byte {@code byteIndex} from a blob using nibble-level access. */ + static byte byteAt(ABlob blob, int byteIndex) { + int ni = byteIndex * 2; + return (byte)((blob.getHexDigit(ni) << 4) | blob.getHexDigit(ni + 1)); + } +} diff --git a/convex-db/src/main/java/convex/db/lattice/RowBlock.java b/convex-db/src/main/java/convex/db/lattice/RowBlock.java new file mode 100644 index 000000000..675214d69 --- /dev/null +++ b/convex-db/src/main/java/convex/db/lattice/RowBlock.java @@ -0,0 +1,492 @@ +package convex.db.lattice; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.BiConsumer; + +import convex.core.data.AArrayBlob; +import convex.core.data.ABlob; +import convex.core.data.ACell; +import convex.core.data.AVector; +import convex.core.data.Blob; +import convex.core.data.Vectors; +import convex.core.data.prim.CVMLong; + +/** + * Utility class for prefix-keyed row blocks. + * + *

Since v4, a block is stored as a flat {@link Blob} with binary layout: + * 

+ *   [4 bytes big-endian: count N]
+ *   [N × 4 bytes: entry byte-offsets from start of DATA section]
+ *   DATA section:
+ *     entry[i]:
+ *       [1 byte: pk_len]
+ *       [pk_len bytes: pk]
+ *       [1 byte: flags — 0=live, 1=tombstone]
+ *       [8 bytes: utime (write-sequence number, big-endian long)]
+ *       if flags==0:
+ *         [2 bytes: values_len]
+ *         [values_len bytes: CAD3-encoded values]
+ *
+ * + *

Legacy v1 AVector format ({@code [CVMLong(N), pk0, row0, ...]}) is read-only + * for backward compatibility; writes always produce flat Blobs. + * + *

Block key in the outer Index = first {@link #PREFIX_BYTES} bytes of the PK. + * Default is 8, increased from 6 to avoid degenerate single-block scenarios with + * sequential integer PKs. + */ +public class RowBlock { + + /** Number of PK prefix bytes used as the block's Index key. */ + public static final int PREFIX_BYTES = Integer.getInteger("convex.block.prefix", 8); + + /** Byte size of the flat block header (just the count field). */ + private static final int HDR = 4; + + private RowBlock() {} + + // ── Low-level int helpers ───────────────────────────────────────────────── + + private static int rInt(byte[] b, int off) { + return ((b[off]&0xFF)<<24)|((b[off+1]&0xFF)<<16)|((b[off+2]&0xFF)<<8)|(b[off+3]&0xFF); + } + + private static void wInt(byte[] b, int off, int v) { + b[off]=(byte)(v>>>24); b[off+1]=(byte)(v>>>16); b[off+2]=(byte)(v>>>8); b[off+3]=(byte)v; + } + + /** Absolute byte position of the DATA section for a block with N entries. */ + private static int dataStart(int n) { return HDR + 4 * n; } + + // ── Block key ───────────────────────────────────────────────────────────── + + /** + * Returns the block Index key for a given primary key: the first + * {@link #PREFIX_BYTES} bytes of {@code pk}, or {@code pk} itself if shorter. + */ + public static ABlob blockKey(ABlob pk) { + int plen = (int) pk.count(); + if (plen <= PREFIX_BYTES) return pk; + byte[] bytes = new byte[PREFIX_BYTES]; + for (int i = 0; i < PREFIX_BYTES; i++) bytes[i] = pk.byteAtUnchecked(i); + return Blob.wrap(bytes); + } + + // ── Detection ───────────────────────────────────────────────────────────── + + /** + * Returns true if block is a RowBlock (flat Blob v4 or legacy AVector v1). + */ + public static boolean isBlock(ACell block) { + if (block instanceof Blob) return true; + if (!(block instanceof AVector)) return false; + @SuppressWarnings("unchecked") AVector v = (AVector) block; + if (v.count() < 1) return false; + ACell first = v.get(0); + if (!(first instanceof CVMLong)) return false; + long n = ((CVMLong) first).longValue(); + if (n < 0) return false; + return v.count() == 1 + 2 * n; + } + + // ── Count ───────────────────────────────────────────────────────────────── + + /** Number of rows (live + tombstone) in this block. Returns 0 for null or non-block values. */ + public static int count(ACell block) { + if (block instanceof AArrayBlob) { + AArrayBlob b = (AArrayBlob) block; + if (b.count() < 4) return 0; + byte[] bs = b.getInternalArray(); + int base = b.getInternalOffset(); + return rInt(bs, base); + } + if (!(block instanceof AVector)) return 0; + @SuppressWarnings("unchecked") AVector v = (AVector) block; + if (!isBlock(v)) return 0; + return (int)((CVMLong)v.get(0)).longValue(); + } + + // ── pk-comparison on raw bytes ──────────────────────────────────────────── + + /** Compare the stored pk at entry position entryStart (in bs) with the given pk. */ + private static int cmpAt(byte[] bs, int entryStart, ABlob pk) { + int sLen = bs[entryStart] & 0xFF; + int gLen = (int) pk.count(); + int min = Math.min(sLen, gLen); + for (int i = 0; i < min; i++) { + int c = (bs[entryStart+1+i]&0xFF) - (pk.byteAtUnchecked(i)&0xFF); + if (c != 0) return c; + } + return sLen - gLen; + } + + /** + * Binary search in the full flat-block byte array. + * @param base absolute offset of the block's first byte within bs + * @return index if found, or -(insertionPoint+1) if not found + */ + private static int bsearch(byte[] bs, int n, int base, int ds, ABlob pk) { + int lo = 0, hi = n - 1; + while (lo <= hi) { + int mid = (lo + hi) >>> 1; + int off = rInt(bs, base + HDR + 4 * mid); + int cmp = cmpAt(bs, ds + off, pk); + if (cmp == 0) return mid; + if (cmp < 0) lo = mid + 1; else hi = mid - 1; + } + return -(lo + 1); + } + + // ── Row decode from flat bytes ──────────────────────────────────────────── + + /** + * Decode a SQLRow from flat bytes. afterPkPos points to: flags(1) utime(8) [valLen(2) val(*)]. + */ + private static AVector decodeRow(byte[] bs, int afterPkPos) { + int flags = bs[afterPkPos] & 0xFF; + byte[] u = new byte[8]; + System.arraycopy(bs, afterPkPos + 1, u, 0, 8); + Blob utime = Blob.wrap(u); + if (flags != 0) return Vectors.of(null, utime, utime); // tombstone + int vLen = ((bs[afterPkPos+9]&0xFF)<<8)|(bs[afterPkPos+10]&0xFF); + byte[] v = new byte[vLen]; + System.arraycopy(bs, afterPkPos + 11, v, 0, vLen); + return Vectors.of(Blob.wrap(v), utime); + } + + // ── Entry encoder ───────────────────────────────────────────────────────── + + private static byte[] encodeEntry(ABlob pk, AVector row) { + int pkLen = (int) pk.count(); + boolean live = SQLRow.isLive(row); + + byte[] pkB = new byte[pkLen]; + pk.getBytes(pkB, 0); + + // utime (8 bytes in flat format; normalise legacy 4-byte blobs and CVMLong) + byte[] utB = new byte[8]; + ACell utCell = row.get(SQLRow.POS_UTIME); + if (utCell instanceof ABlob) { + ABlob utBlob = (ABlob) utCell; + if (utBlob.count() == 8) { + utBlob.getBytes(utB, 0); + } else { + // legacy 4-byte compact seconds → decode and re-encode as 8-byte long + long ms = SQLRow.decodeTimestampMs(utBlob); + convex.core.util.Utils.writeLong(utB, 0, ms); + } + } else if (utCell instanceof CVMLong) { + convex.core.util.Utils.writeLong(utB, 0, ((CVMLong)utCell).longValue()); + } + + byte[] vB = new byte[0]; + if (live) { + ACell vc = row.get(SQLRow.POS_VALUES); + if (vc instanceof ABlob) { + int vl = (int)((ABlob)vc).count(); + vB = new byte[vl]; + ((ABlob)vc).getBytes(vB, 0); + } else if (vc instanceof AVector) { + @SuppressWarnings("unchecked") Blob enc = SQLRow.encodeValues((AVector)vc); + vB = new byte[(int)enc.count()]; + enc.getBytes(vB, 0); + } + } + + int sz = 1 + pkLen + 1 + 8 + (live ? 2 + vB.length : 0); + byte[] e = new byte[sz]; + int p = 0; + e[p++] = (byte) pkLen; + System.arraycopy(pkB, 0, e, p, pkLen); p += pkLen; + e[p++] = live ? (byte)0 : (byte)1; + System.arraycopy(utB, 0, e, p, 8); p += 8; + if (live) { + e[p++] = (byte)(vB.length>>>8); e[p++] = (byte)vB.length; + System.arraycopy(vB, 0, e, p, vB.length); + } + return e; + } + + // ── Flat Blob builder ───────────────────────────────────────────────────── + + private static Blob buildFlat(List pks, List> rows) { + int n = pks.size(); + byte[][] enc = new byte[n][]; + int total = 0; + for (int i = 0; i < n; i++) { + enc[i] = encodeEntry(pks.get(i), rows.get(i)); + total += enc[i].length; + } + byte[] bs = new byte[HDR + 4*n + total]; + wInt(bs, 0, n); + int dp = HDR + 4*n, off = 0; + for (int i = 0; i < n; i++) { + wInt(bs, HDR + 4*i, off); + System.arraycopy(enc[i], 0, bs, dp, enc[i].length); + dp += enc[i].length; + off += enc[i].length; + } + return Blob.wrap(bs); + } + + // ── Extract all entries ─────────────────────────────────────────────────── + + @SuppressWarnings("unchecked") + private static void extractAll(ACell block, List pksOut, List> rowsOut) { + if (block instanceof AArrayBlob) { + AArrayBlob blob = (AArrayBlob) block; + byte[] bs = blob.getInternalArray(); + int base = blob.getInternalOffset(); + int n = rInt(bs, base); + int ds = base + dataStart(n); + for (int i = 0; i < n; i++) { + int off = rInt(bs, base + HDR + 4*i); + int es = ds + off; + int pkLen = bs[es] & 0xFF; + byte[] pkB = new byte[pkLen]; + System.arraycopy(bs, es+1, pkB, 0, pkLen); + pksOut.add(Blob.wrap(pkB)); + rowsOut.add(decodeRow(bs, es + 1 + pkLen)); + } + return; + } + if (!(block instanceof AVector)) return; + AVector v = (AVector) block; + if (!isBlock(v)) return; + int n = (int)((CVMLong)v.get(0)).longValue(); + for (int i = 0; i < n; i++) { + pksOut.add((ABlob) v.get(1 + 2*i)); + rowsOut.add((AVector) v.get(2 + 2*i)); + } + } + + // ── Point lookup ────────────────────────────────────────────────────────── + + /** + * Finds a row by primary key. Returns SQLRow-format entry, or null if not found. + */ + @SuppressWarnings("unchecked") + public static AVector get(ACell block, ABlob pk) { + if (block instanceof AArrayBlob) { + AArrayBlob blob = (AArrayBlob) block; + if (blob.count() < 4) return null; + byte[] bs = blob.getInternalArray(); + int base = blob.getInternalOffset(); + int n = rInt(bs, base); + if (n == 0) return null; + int ds = base + dataStart(n); + int idx = bsearch(bs, n, base, ds, pk); + if (idx < 0) return null; + int off = rInt(bs, base + HDR + 4*idx); + int pkLen = bs[ds+off] & 0xFF; + return decodeRow(bs, ds + off + 1 + pkLen); + } + if (!(block instanceof AVector)) return null; + AVector v = (AVector) block; + if (!isBlock(v)) return null; + int n = (int)((CVMLong)v.get(0)).longValue(); + int lo = 0, hi = n - 1; + while (lo <= hi) { + int mid = (lo + hi) >>> 1; + int cmp = ((ABlob)v.get(1+2*mid)).compareTo(pk); + if (cmp == 0) return (AVector) v.get(2+2*mid); + if (cmp < 0) lo = mid + 1; else hi = mid - 1; + } + return null; + } + + // ── Indexed access ──────────────────────────────────────────────────────── + + /** Returns the pk at position i (0-based). */ + @SuppressWarnings("unchecked") + public static ABlob getKey(ACell block, int i) { + if (block instanceof AArrayBlob) { + AArrayBlob blob = (AArrayBlob) block; + byte[] bs = blob.getInternalArray(); + int base = blob.getInternalOffset(); + int n = rInt(bs, base); + int ds = base + dataStart(n); + int off = rInt(bs, base + HDR + 4*i); + int pkLen = bs[ds+off] & 0xFF; + byte[] pkB = new byte[pkLen]; + System.arraycopy(bs, ds+off+1, pkB, 0, pkLen); + return Blob.wrap(pkB); + } + AVector v = (AVector) block; + return (ABlob) v.get(1 + 2*i); + } + + /** Returns the row entry at position i (0-based). */ + @SuppressWarnings("unchecked") + public static AVector getRow(ACell block, int i) { + if (block instanceof AArrayBlob) { + AArrayBlob blob = (AArrayBlob) block; + byte[] bs = blob.getInternalArray(); + int base = blob.getInternalOffset(); + int n = rInt(bs, base); + int ds = base + dataStart(n); + int off = rInt(bs, base + HDR + 4*i); + int es = ds + off; + int pkLen = bs[es] & 0xFF; + return decodeRow(bs, es + 1 + pkLen); + } + AVector v = (AVector) block; + return (AVector) v.get(2 + 2*i); + } + + // ── Single-entry mutation ───────────────────────────────────────────────── + + /** + * Adds or replaces a row entry in the block, maintaining sorted pk order. + * Creates a new single-entry flat block if block is null or not a block. + * + * @return New block as flat Blob with the row inserted or updated + */ + public static ACell put(ACell block, ABlob pk, AVector row) { + if (!isBlock(block)) { + // null or raw SQLRow — create single-entry flat block + List pks = new ArrayList<>(1); + List> rows = new ArrayList<>(1); + pks.add(pk); rows.add(row); + return buildFlat(pks, rows); + } + List pks = new ArrayList<>(count(block) + 1); + List> rows = new ArrayList<>(count(block) + 1); + extractAll(block, pks, rows); + // binary search for pk + int lo = 0, hi = pks.size() - 1, pos = pks.size(); + while (lo <= hi) { + int mid = (lo + hi) >>> 1; + int cmp = pks.get(mid).compareTo(pk); + if (cmp == 0) { rows.set(mid, row); return buildFlat(pks, rows); } + if (cmp < 0) lo = mid + 1; else { pos = mid; hi = mid - 1; } + } + pks.add(pos, pk); + rows.add(pos, row); + return buildFlat(pks, rows); + } + + // ── Batch mutation ──────────────────────────────────────────────────────── + + /** + * Merges a sorted list of new (pk → row) entries into the block. + * New entries override existing entries with the same pk. + * newPks must be sorted ascending and same size as newRows. + * + * @param newLiveCount if non-null, newLiveCount[0] is incremented for each newly-live row + * @return new block as flat Blob + */ + public static ACell putAll(ACell block, + List newPks, List> newRows, + int[] newLiveCount) { + List ePks = new ArrayList<>(); + List> eRows = new ArrayList<>(); + if (isBlock(block)) extractAll(block, ePks, eRows); + + int en = ePks.size(), nn = newPks.size(); + List rPks = new ArrayList<>(en + nn); + List> rRows = new ArrayList<>(en + nn); + + int ei = 0, ni = 0; + while (ei < en && ni < nn) { + int cmp = ePks.get(ei).compareTo(newPks.get(ni)); + if (cmp < 0) { + rPks.add(ePks.get(ei)); rRows.add(eRows.get(ei)); ei++; + } else if (cmp > 0) { + if (newLiveCount != null && SQLRow.isLive(newRows.get(ni))) newLiveCount[0]++; + rPks.add(newPks.get(ni)); rRows.add(newRows.get(ni)); ni++; + } else { + // same pk — new wins; track live transition + AVector existingRow = eRows.get(ei); + AVector newRow = newRows.get(ni); + if (newLiveCount != null && !SQLRow.isLive(existingRow) && SQLRow.isLive(newRow)) { + newLiveCount[0]++; + } + rPks.add(newPks.get(ni)); rRows.add(newRow); ei++; ni++; + } + } + while (ei < en) { rPks.add(ePks.get(ei)); rRows.add(eRows.get(ei)); ei++; } + while (ni < nn) { + if (newLiveCount != null && SQLRow.isLive(newRows.get(ni))) newLiveCount[0]++; + rPks.add(newPks.get(ni)); rRows.add(newRows.get(ni)); ni++; + } + return buildFlat(rPks, rRows); + } + + // ── Iteration ───────────────────────────────────────────────────────────── + + /** + * Iterates all (pk, rowEntry) pairs in pk order. + */ + @SuppressWarnings("unchecked") + public static void forEach(ACell block, BiConsumer> action) { + if (block instanceof AArrayBlob) { + AArrayBlob blob = (AArrayBlob) block; + byte[] bs = blob.getInternalArray(); + int base = blob.getInternalOffset(); + int n = rInt(bs, base); + int ds = base + dataStart(n); + for (int i = 0; i < n; i++) { + int off = rInt(bs, base + HDR + 4*i); + int es = ds + off; + int pkLen = bs[es] & 0xFF; + byte[] pkB = new byte[pkLen]; + System.arraycopy(bs, es+1, pkB, 0, pkLen); + action.accept(Blob.wrap(pkB), decodeRow(bs, es + 1 + pkLen)); + } + return; + } + if (!(block instanceof AVector)) return; + AVector v = (AVector) block; + if (!isBlock(v)) return; + int n = (int)((CVMLong)v.get(0)).longValue(); + for (int i = 0; i < n; i++) { + action.accept((ABlob)v.get(1+2*i), (AVector)v.get(2+2*i)); + } + } + + // ── Merge ───────────────────────────────────────────────────────────────── + + /** + * Merges two blocks covering the same key prefix using row-level LWW semantics. + */ + public static ACell merge(ACell a, ACell b) { + if (!isBlock(a)) return b; + if (!isBlock(b)) return a; + if (a == b) return a; // identical reference — no work needed + + List aPks = new ArrayList<>(), bPks = new ArrayList<>(); + List> aRows = new ArrayList<>(), bRows = new ArrayList<>(); + extractAll(a, aPks, aRows); + extractAll(b, bPks, bRows); + + int na = aPks.size(), nb = bPks.size(); + List rPks = new ArrayList<>(na + nb); + List> rRows = new ArrayList<>(na + nb); + + boolean changed = false; + int i = 0, j = 0; + while (i < na && j < nb) { + int cmp = aPks.get(i).compareTo(bPks.get(j)); + if (cmp < 0) { rPks.add(aPks.get(i)); rRows.add(aRows.get(i++)); } + else if (cmp > 0) { + changed = true; + rPks.add(bPks.get(j)); rRows.add(bRows.get(j++)); + } else { + AVector ar = aRows.get(i), br = bRows.get(j); + AVector merged = SQLRow.merge(ar, br); + if (merged != ar) changed = true; + rPks.add(aPks.get(i)); + rRows.add(merged); + i++; j++; + } + } + while (i < na) { rPks.add(aPks.get(i)); rRows.add(aRows.get(i++)); } + while (j < nb) { changed = true; rPks.add(bPks.get(j)); rRows.add(bRows.get(j++)); } + + if (!changed) return a; // a dominates — return without re-encoding + return buildFlat(rPks, rRows); + } +} diff --git a/convex-db/src/main/java/convex/db/lattice/RowBlockLattice.java b/convex-db/src/main/java/convex/db/lattice/RowBlockLattice.java new file mode 100644 index 000000000..785f95e53 --- /dev/null +++ b/convex-db/src/main/java/convex/db/lattice/RowBlockLattice.java @@ -0,0 +1,36 @@ +package convex.db.lattice; + +import convex.core.data.ACell; +import convex.lattice.ALattice; + +/** + * Lattice for individual row block entries. + * Merges two blocks covering the same prefix key using {@link RowBlock#merge}. + */ +public class RowBlockLattice extends ALattice { + + public static final RowBlockLattice INSTANCE = new RowBlockLattice(); + + private RowBlockLattice() {} + + @Override + public ACell zero() { return null; } + + @Override + public ACell merge(ACell a, ACell b) { + if (a == null) return b; + if (b == null) return a; + return RowBlock.merge(a, b); + } + + @Override + public boolean checkForeign(ACell value) { + return RowBlock.isBlock(value); + } + + @SuppressWarnings("unchecked") + @Override + public ALattice path(ACell childKey) { + return null; // blocks are leaf nodes + } +} diff --git a/convex-db/src/main/java/convex/db/lattice/SQLRow.java b/convex-db/src/main/java/convex/db/lattice/SQLRow.java index 402847288..df8f4ab8b 100644 --- a/convex-db/src/main/java/convex/db/lattice/SQLRow.java +++ b/convex-db/src/main/java/convex/db/lattice/SQLRow.java @@ -1,56 +1,133 @@ package convex.db.lattice; +import convex.core.data.ABlob; import convex.core.data.ACell; import convex.core.data.AVector; +import convex.core.data.Blob; +import convex.core.data.CAD3Encoder; +import convex.core.data.Cells; import convex.core.data.Vectors; import convex.core.data.prim.CVMLong; +import convex.core.exceptions.BadFormatException; +import convex.core.util.Utils; /** * Utility class for SQL row entries in the table lattice. * - *

A row entry is a vector: [values, utime, deleted] + *

Current row format (v4): *

    - *
  • values (AVector) - Column values for this row
    • - *
  • utime (CVMLong) - Update timestamp for LWW conflict resolution
    • - *
  • deleted (CVMLong) - Deletion timestamp, or null if live
    • + *
  • Live row — 2-element vector: [blob_values, version_blob8]
    • + *
  • Tombstone — 3-element vector: [null, version_blob8, version_blob8]
    • *
+ * {@code blob_values} is the standard CAD3 encoding of the column-values AVector, + * stored as a single {@link Blob}. This reduces per-row heap cost by ~4× compared + * to storing the AVector and its element cells separately. * - *

Merge semantics: Latest timestamp wins. If timestamps equal, deletion wins. + *

{@code version_blob8} is an 8-byte {@link Blob} holding a monotonic write + * sequence number supplied by {@link SQLSchema}. The counter is initialised from + * {@code System.currentTimeMillis()} at schema creation time and incremented + * atomically on every write, so every row gets a unique version — no two writes + * within a single schema instance can tie. + * + *

Legacy format (v3, read-only): [blob_values, utime_blob4] — 4-byte compact + * seconds since {@link #COMPACT_EPOCH_S}. Values decode to ~1.7e12, which is + * below any v4 counter (also initialised near 1.7e12 from currentTimeMillis), + * so old rows sort before new rows in LWW comparisons. + * Legacy format (v2, read-only): [AVector(values), utime_blob4]. + * Legacy format (v1, read-only): [AVector(values), CVMLong_ms, CVMLong_ms_or_null]. + * {@link #getVersion} handles all formats transparently. + * + *

Merge semantics: highest version wins; equal versions → deletion wins. */ public class SQLRow { - /** Position of values in the row vector */ - static final int POS_VALUES = 0; - /** Position of update timestamp */ - static final int POS_UTIME = 1; - /** Position of deletion timestamp */ + /** Position of column values in the row vector */ + static final int POS_VALUES = 0; + /** Position of version / write sequence number */ + static final int POS_UTIME = 1; + /** Position of deletion version (tombstones only) */ static final int POS_DELETED = 2; + /** Unix seconds for 2020-01-01 00:00:00 UTC — used only for v3 legacy decode. */ + static final long COMPACT_EPOCH_S = 1_577_836_800L; + private SQLRow() {} + // ── Version codec ─────────────────────────────────────────────────────── + + /** + * Encodes a write-sequence version as an 8-byte Blob (v4 format). + * The {@code version} value is the raw long from {@link SQLSchema}'s write counter. + */ + static Blob encodeTimestamp(long version) { + byte[] bs = new byte[8]; + Utils.writeLong(bs, 0, version); + return Blob.wrap(bs); + } + + /** + * Decodes the version from a row's timestamp blob. + * Handles v4 (8-byte, raw long), v3 (4-byte compact seconds), and + * v1/v2 (CVMLong milliseconds) transparently. + */ + static long decodeTimestampMs(ABlob blob) { + if (blob.count() == 8) return blob.longValue(); // v4: raw sequence number + long s = blob.longValue() & 0xFFFFFFFFL; // v3: unsigned 32-bit compact seconds + return (COMPACT_EPOCH_S + s) * 1000L; + } + + // ── Values codec (v3) ────────────────────────────────────────────────── + /** - * Creates a new live row entry with the given values. + * Encodes column values to a compact Blob using standard CAD3 format. + * For a 4-column row the Blob is typically 30–50 bytes vs ~160 bytes for + * the equivalent AVector + child cells. + */ + static Blob encodeValues(AVector values) { + return Cells.encode(values); + } + + /** + * Decodes column values from a compact Blob (v3 format). + */ + @SuppressWarnings("unchecked") + static AVector decodeValues(Blob blob) { + try { + return (AVector) CAD3Encoder.INSTANCE.decode(blob); + } catch (BadFormatException e) { + throw new IllegalStateException("Compact row decode failed", e); + } + } + + // ── Factory methods ──────────────────────────────────────────────────── + + /** + * Creates a new live row entry (v3 compact format). * - * @param values Column values for the row - * @param timestamp Update timestamp - * @return Row entry vector + * @param values Column values for the row + * @param timestamp Update timestamp (milliseconds) + * @return 2-element row vector [blob_values, utime_blob4] */ public static AVector create(AVector values, CVMLong timestamp) { - return Vectors.of(values, timestamp, null); + return Vectors.of(encodeValues(values), encodeTimestamp(timestamp.longValue())); } /** * Creates a tombstone entry for a deleted row. * - * @param timestamp Deletion timestamp - * @return Tombstone row entry + * @param timestamp Deletion timestamp (milliseconds) + * @return 3-element tombstone vector [null, utime_blob4, deleted_blob4] */ public static AVector createTombstone(CVMLong timestamp) { - return Vectors.of(null, timestamp, timestamp); + Blob ts = encodeTimestamp(timestamp.longValue()); + return Vectors.of(null, ts, ts); } + // ── Accessors ────────────────────────────────────────────────────────── + /** * Gets the column values from a row entry. + * Handles v3 (Blob-encoded), v2 (AVector+Blob4), and v1 (AVector+CVMLong). * * @param row Row entry vector * @return Column values, or null if tombstone @@ -58,40 +135,50 @@ public static AVector createTombstone(CVMLong timestamp) { @SuppressWarnings("unchecked") public static AVector getValues(AVector row) { if (row == null) return null; - return (AVector) row.get(POS_VALUES); + ACell cell = row.get(POS_VALUES); + if (cell instanceof Blob blob) return decodeValues(blob); // v3 compact + return (AVector) cell; // v1/v2 legacy } /** - * Gets the update timestamp from a row entry. + * Gets the update timestamp as a CVMLong (milliseconds). + * Handles both compact Blob(4) format (v2) and legacy CVMLong format (v1). * * @param row Row entry vector - * @return Update timestamp + * @return Update timestamp in milliseconds, or null */ public static CVMLong getTimestamp(AVector row) { if (row == null) return null; - return (CVMLong) row.get(POS_UTIME); + ACell cell = row.get(POS_UTIME); + if (cell instanceof CVMLong) return (CVMLong) cell; // v1 legacy + if (cell instanceof ABlob) return CVMLong.create(decodeTimestampMs((ABlob) cell)); // v2 + return null; } /** - * Gets the deletion timestamp from a row entry. + * Gets the deletion timestamp from a tombstone row entry. * * @param row Row entry vector - * @return Deletion timestamp, or null if live + * @return Deletion timestamp in milliseconds, or null if live */ public static CVMLong getDeleted(AVector row) { - if (row == null) return null; - return (CVMLong) row.get(POS_DELETED); + if (row == null || row.count() < 3) return null; + ACell cell = row.get(POS_DELETED); + if (cell instanceof CVMLong) return (CVMLong) cell; + if (cell instanceof ABlob) return CVMLong.create(decodeTimestampMs((ABlob) cell)); + return null; } /** * Checks if a row entry is a tombstone (deleted). + * Tombstones always have 3 elements; live rows have 2. * * @param row Row entry vector * @return true if tombstone */ public static boolean isTombstone(AVector row) { if (row == null) return false; - return row.get(POS_DELETED) != null; + return row.count() == 3; } /** @@ -105,17 +192,19 @@ public static boolean isLive(AVector row) { } /** - * Updates a row entry with new values, preserving the timestamp. + * Updates a row entry with new values and a new timestamp. * - * @param row Original row entry - * @param values New column values - * @param timestamp Update timestamp - * @return Updated row entry + * @param row Original row entry (unused; kept for API symmetry) + * @param values New column values + * @param timestamp New update timestamp (milliseconds) + * @return Updated live row entry */ public static AVector withValues(AVector row, AVector values, CVMLong timestamp) { - return Vectors.of(values, timestamp, null); + return Vectors.of(encodeValues(values), encodeTimestamp(timestamp.longValue())); } + // ── Merge ────────────────────────────────────────────────────────────── + /** * Merges two row entries using LWW semantics. * Latest timestamp wins. If equal, deletion wins. diff --git a/convex-db/src/main/java/convex/db/lattice/SQLSchema.java b/convex-db/src/main/java/convex/db/lattice/SQLSchema.java index 23b87c3cf..403e6ce48 100644 --- a/convex-db/src/main/java/convex/db/lattice/SQLSchema.java +++ b/convex-db/src/main/java/convex/db/lattice/SQLSchema.java @@ -1,5 +1,10 @@ package convex.db.lattice; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + import convex.core.data.ABlob; import convex.core.data.ACell; import convex.core.data.AString; @@ -46,6 +51,16 @@ */ public class SQLSchema extends ALatticeComponent>> { + /** + * JVM-global monotonic write-sequence counter. Initialised from + * {@code System.currentTimeMillis()} so that v4 sequence numbers always sort + * after v3 compact-timestamp rows (~1.7e12) in LWW comparisons. + * Being static means every write in any schema instance within this process gets + * a unique version — no ties are possible even when multiple forks write + * concurrently within the same millisecond. + */ + private static final AtomicLong WRITE_SEQ = new AtomicLong(System.currentTimeMillis()); + public SQLSchema(ALatticeCursor>> cursor) { super(cursor); } @@ -82,8 +97,9 @@ public SQLSchema fork() { // ========== Internal Helpers ========== + /** Returns the next unique write-sequence number for LWW ordering. */ private CVMLong now() { - return CVMLong.create(System.currentTimeMillis()); + return CVMLong.create(WRITE_SEQ.incrementAndGet()); } /** @@ -122,7 +138,7 @@ public SQLTable getLiveTable(String name) { * @return ABlob representation * @throws IllegalArgumentException if key type not supported */ - private ABlob toKey(ACell key) { + protected ABlob toKey(ACell key) { if (key instanceof ABlob blob) return blob; if (key instanceof CVMLong n) { // Encode as 8-byte big-endian @@ -182,21 +198,36 @@ public boolean createTable(AString name, String[] columns, ConvexColumnType[] ty throw new IllegalArgumentException("Columns and types must have same length"); } - if (getLiveTable(name) != null) return false; - - // Build schema: [[name, typeName, precision, scale], ...] - AVector schema = Vectors.empty(); + // Build full desired schema: [[name, typeName, precision, scale], ...] + AVector newSchema = Vectors.empty(); for (int i = 0; i < columns.length; i++) { ConvexColumnType ct = types[i]; AString typeName = (ct.getBaseType() == ConvexType.ANY) ? null : Strings.create(ct.getBaseType().name()); CVMLong precision = ct.hasPrecision() ? CVMLong.create(ct.getPrecision()) : null; CVMLong scale = ct.hasScale() ? CVMLong.create(ct.getScale()) : null; - schema = schema.append(Vectors.of(Strings.create(columns[i]), typeName, precision, scale)); + newSchema = newSchema.append(Vectors.of(Strings.create(columns[i]), typeName, precision, scale)); } - // Set state on the table's cursor path + SQLTable existing = getLiveTable(name); + if (existing == null) { + // Table does not exist yet: create fresh + ALatticeCursor> tableCursor = cursor.path(name); + tableCursor.set(SQLTable.createState((AVector>) newSchema, now())); + return true; + } + + // Table already exists: append any new columns not present in stored schema + AVector> existingSchema = existing.getSchema(); + long existingCount = existingSchema != null ? existingSchema.count() : 0; + if (newSchema.count() <= existingCount) return false; // no new columns to add + + AVector combinedSchema = existingSchema; + for (long i = existingCount; i < newSchema.count(); i++) { + combinedSchema = combinedSchema.append(newSchema.get(i)); + } + final AVector> finalSchema = (AVector>) combinedSchema; ALatticeCursor> tableCursor = cursor.path(name); - tableCursor.set(SQLTable.createState((AVector>) schema, now())); + tableCursor.updateAndGet(state -> state.assoc(SQLTable.POS_SCHEMA, finalSchema)); return true; } @@ -318,6 +349,34 @@ public boolean insert(String tableName, Object... values) { return insert(Strings.create(tableName), Vectors.of(values)); } + /** + * Batch-inserts rows into a table in a single atomic lattice update. + * + *

Rows are sorted by primary key, grouped by block-key prefix, and written + * with one {@code Index.assoc()} per block rather than one per row. This + * reduces intermediate allocation from O(N × trie-depth) to O(blocks × trie-depth). + * + * @param tableName Target table (must exist and be live) + * @param rows Rows to insert; first element of each row is the primary key + * @return number of newly-live rows added + */ + public int insertAll(String tableName, List> rows) { + return insertAll(Strings.create(tableName), rows); + } + + public int insertAll(AString tableName, List> rows) { + if (rows == null || rows.isEmpty()) return 0; + SQLTable table = getLiveTable(tableName); + if (table == null) return 0; + CVMLong ts = now(); + List>> sorted = new ArrayList<>(rows.size()); + for (AVector row : rows) { + sorted.add(Map.entry(toKey(row.get(0)), row)); + } + sorted.sort(Map.Entry.comparingByKey()); + return table.insertRows(sorted, ts); + } + /** Selects a row by primary key. */ public AVector selectByKey(String tableName, ACell primaryKey) { return selectByKey(Strings.create(tableName), primaryKey); @@ -327,11 +386,13 @@ public AVector selectByKey(AString tableName, ACell primaryKey) { SQLTable table = getLiveTable(tableName); if (table == null) return null; - Index> rows = table.getRows(); + Index rows = table.getRows(); if (rows == null) return null; - ABlob key = toKey(primaryKey); - AVector row = rows.get(key); + ABlob pk = toKey(primaryKey); + ABlob bk = RowBlock.blockKey(pk); + ACell block = rows.get(bk); + AVector row = RowBlock.get(block, pk); if (row == null || !SQLRow.isLive(row)) return null; return SQLRow.getValues(row); } @@ -358,14 +419,19 @@ public Index> selectAll(AString tableName) { SQLTable table = getLiveTable(tableName); if (table == null) return Index.none(); - Index> rows = table.getRows(); + Index rows = table.getRows(); if (rows == null) return Index.none(); - // Single-pass tree traversal: filter tombstones, unwrap values + // Iterate blocks, expand live rows keyed by full pk Index>[] result = new Index[] { Index.none() }; - rows.forEach((k, v) -> { - if (SQLRow.isLive(v)) { - result[0] = result[0].assoc(k, SQLRow.getValues(v)); + rows.forEach((bk, block) -> { + if (RowBlock.isBlock(block)) { + RowBlock.forEach(block, (pk, row) -> { + if (SQLRow.isLive(row)) result[0] = result[0].assoc(pk, SQLRow.getValues(row)); + }); + } else if (block instanceof AVector && SQLRow.isLive((AVector)block)) { + // Legacy single-row entry (backward compat) + result[0] = result[0].assoc(bk, SQLRow.getValues((AVector)block)); } }); return result[0]; @@ -399,4 +465,208 @@ public int getColumnCount(AString name) { if (table == null) return 0; return (int) table.getColumnCount(); } + + // ========== Secondary Index Operations ========== + + /** + * Creates a secondary index on a column. + * Scans existing rows to build the initial index, then maintains it on + * subsequent inserts and deletes. + * + * @param tableName Target table (must exist and be live) + * @param columnName Column to index (must exist in the table schema) + * @return true if the index was created; false if it already exists, + * the table doesn't exist, or the column doesn't exist + */ + public boolean createIndex(String tableName, String columnName) { + return createIndex(Strings.create(tableName), Strings.create(columnName)); + } + + public boolean createIndex(AString tableName, AString columnName) { + SQLTable table = getLiveTable(tableName); + if (table == null) return false; + return table.createColumnIndex(columnName); + } + + /** + * Returns true if a secondary index exists on the named column. + */ + public boolean hasIndex(String tableName, String columnName) { + return hasIndex(Strings.create(tableName), Strings.create(columnName)); + } + + public boolean hasIndex(AString tableName, AString columnName) { + SQLTable table = getLiveTable(tableName); + if (table == null) return false; + return table.hasColumnIndex(columnName); + } + + /** + * Drops a secondary index on a column. + * + * @return true if the index was dropped; false if it didn't exist + */ + public boolean dropIndex(String tableName, String columnName) { + return dropIndex(Strings.create(tableName), Strings.create(columnName)); + } + + public boolean dropIndex(AString tableName, AString columnName) { + SQLTable table = getLiveTable(tableName); + if (table == null) return false; + return table.dropColumnIndex(columnName); + } + + /** + * Returns all live rows where the named column equals {@code value}. + * + *

If a secondary index exists on the column, uses the index to avoid a + * full-table scan. Otherwise falls back to scanning all rows. + * + *

Results are always re-validated against the row store, so stale index + * entries (possible after distributed merge) are silently filtered out. + * + * @param tableName Target table + * @param columnName Column to filter on + * @param value Value to match (must be the same ACell type as stored) + * @return Index of matching rows keyed by primary-key blob, or empty if none + */ + public Index> selectByColumn( + String tableName, String columnName, ACell value) { + return selectByColumn(Strings.create(tableName), Strings.create(columnName), value); + } + + @SuppressWarnings("unchecked") + public Index> selectByColumn( + AString tableName, AString columnName, ACell value) { + SQLTable table = getLiveTable(tableName); + if (table == null) return Index.none(); + + // Try index-backed lookup first + Index> allIndices = + SQLTable.getIndicesFromState(table.getState()); + if (allIndices != null) { + ACell rawColIdx = allIndices.get(columnName); + if (rawColIdx instanceof Index) { + Index colIdx = (Index) rawColIdx; + Index>[] result = new Index[]{Index.none()}; + colIdx.forEach((indexKey, pk) -> { + if (!ColumnIndex.matchesValue(indexKey, value)) return; + ABlob pkBlob = ColumnIndex.extractPk(indexKey); + AVector row = table.selectByKeyBlob(pkBlob); + if (row == null) return; // tombstoned or missing + // Re-validate column value (guards against stale index entries) + AVector> schema = table.getSchema(); + int ci = (schema != null) + ? SQLTable.findColIdxInSchema(schema, columnName) : -1; + if (ci >= 0 && ci < (int) row.count() && value.equals(row.get(ci))) { + result[0] = result[0].assoc(pkBlob, row); + } + }); + return result[0]; + } + } + + // Fallback: full-scan with in-memory filter + AVector> schema = table.getSchema(); + int colIdx = (schema != null) + ? SQLTable.findColIdxInSchema(schema, columnName) : -1; + if (colIdx < 0) return Index.none(); + Index> all = selectAll(tableName); + Index>[] result = new Index[]{Index.none()}; + all.forEach((pk, row) -> { + if (colIdx < (int) row.count() && value.equals(row.get(colIdx))) { + result[0] = result[0].assoc(pk, row); + } + }); + return result[0]; + } + + /** + * Returns all live rows where the named column is in the inclusive range [from, to]. + * + *

Uses the secondary index if present (iterates index entries filtering by range). + * Falls back to a full-table scan otherwise. + * + *

Range comparison uses sortable encoding: CVMLong values compare numerically; + * AString values compare lexicographically by UTF-8 bytes. + * + * @param tableName Target table + * @param columnName Column to filter on + * @param from Lower bound (inclusive) + * @param to Upper bound (inclusive) + * @return Index of matching rows keyed by primary-key blob, or empty if none + */ + public Index> selectByColumnRange( + String tableName, String columnName, ACell from, ACell to) { + return selectByColumnRange( + Strings.create(tableName), Strings.create(columnName), from, to); + } + + @SuppressWarnings("unchecked") + public Index> selectByColumnRange( + AString tableName, AString columnName, ACell from, ACell to) { + SQLTable table = getLiveTable(tableName); + if (table == null) return Index.none(); + + // Try index-backed lookup + Index> allIndices = + SQLTable.getIndicesFromState(table.getState()); + if (allIndices != null) { + ACell rawColIdx = allIndices.get(columnName); + if (rawColIdx instanceof Index) { + Index colIdx = (Index) rawColIdx; + Index>[] result = new Index[]{Index.none()}; + AVector> schema = table.getSchema(); + int ci = (schema != null) + ? SQLTable.findColIdxInSchema(schema, columnName) : -1; + colIdx.forEach((indexKey, pk) -> { + if (!ColumnIndex.matchesRange(indexKey, from, to)) return; + ABlob pkBlob = ColumnIndex.extractPk(indexKey); + AVector row = table.selectByKeyBlob(pkBlob); + if (row == null) return; + // Re-validate (guards against stale index entries) + if (ci >= 0 && ci < (int) row.count()) { + ACell colVal = row.get(ci); + if (ColumnIndex.matchesRange( + ColumnIndex.indexKey(colVal, pkBlob), from, to)) { + result[0] = result[0].assoc(pkBlob, row); + } + } + }); + return result[0]; + } + } + + // Fallback: full-scan with in-memory filter + AVector> schema = table.getSchema(); + int colIdx = (schema != null) + ? SQLTable.findColIdxInSchema(schema, columnName) : -1; + if (colIdx < 0) return Index.none(); + byte[] fromBytes = ColumnIndex.encodeValue(from); + byte[] toBytes = ColumnIndex.encodeValue(to); + Index> all = selectAll(tableName); + Index>[] result = new Index[]{Index.none()}; + all.forEach((pk, row) -> { + if (colIdx >= (int) row.count()) return; + ACell colVal = row.get(colIdx); + if (colVal == null) return; + byte[] valBytes = ColumnIndex.encodeValue(colVal); + if (valBytes.length != fromBytes.length) return; + // Lexicographic comparison + int cmpFrom = 0, cmpTo = 0; + for (int i = 0; i < valBytes.length; i++) { + int b = valBytes[i] & 0xFF; + if (cmpFrom == 0) { + if (b > (fromBytes[i] & 0xFF)) cmpFrom = 1; + else if (b < (fromBytes[i] & 0xFF)) cmpFrom = -1; + } + if (cmpTo == 0) { + if (b < (toBytes[i] & 0xFF)) cmpTo = -1; + else if (b > (toBytes[i] & 0xFF)) cmpTo = 1; + } + } + if (cmpFrom >= 0 && cmpTo <= 0) result[0] = result[0].assoc(pk, row); + }); + return result[0]; + } } diff --git a/convex-db/src/main/java/convex/db/lattice/SQLTable.java b/convex-db/src/main/java/convex/db/lattice/SQLTable.java index 71b834b76..e1771272f 100644 --- a/convex-db/src/main/java/convex/db/lattice/SQLTable.java +++ b/convex-db/src/main/java/convex/db/lattice/SQLTable.java @@ -1,10 +1,16 @@ package convex.db.lattice; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + import convex.core.data.ABlob; import convex.core.data.ACell; import convex.core.data.AString; import convex.core.data.AVector; +import convex.core.data.Blob; import convex.core.data.Index; +import convex.core.data.Strings; import convex.core.data.Vectors; import convex.core.data.prim.CVMLong; import convex.lattice.ALatticeComponent; @@ -13,12 +19,15 @@ /** * A single SQL table within the lattice table store. * - *

Wraps a lattice cursor pointing at the table's state vector: [schema, rows, utime, liveCount] + *

Wraps a lattice cursor pointing at the table's state vector: + * [schema, rows, utime, liveCount, blockVec, indices] *

    *
  • schema (AVector) - Column definitions: [[name, type, precision, scale], ...]
    • - *
  • rows (Index) - Row data: primary-key (ABlob) → row entry
    • + *
  • rows (Index) - Row data: primary-key prefix (ABlob) → RowBlock (flat Blob)
    • *
  • utime (CVMLong) - Schema update timestamp for LWW
    • *
  • liveCount (CVMLong) - Number of live (non-tombstone) rows
    • + *
  • blockVec (AVector|null) - Sequential block list for O(n)-total full scans
    • + *
  • indices (Index) - Column indices: colName (AString) → Index(indexKey → pk)
    • *
* *

Schema is immutable after creation (for now). Row data merges independently. @@ -29,13 +38,33 @@ public class SQLTable extends ALatticeComponent> { /** Position of schema in table vector */ - static final int POS_SCHEMA = 0; + static final int POS_SCHEMA = 0; /** Position of rows in table vector */ - static final int POS_ROWS = 1; + static final int POS_ROWS = 1; /** Position of update timestamp */ - static final int POS_UTIME = 2; + static final int POS_UTIME = 2; /** Position of live row count */ static final int POS_LIVE_COUNT = 3; + /** + * Position of the sequential block vector (for O(1)-heap full scans). + * + *

Value is {@code AVector} (the ordered list of live block blobs) or + * {@code null} when invalidated by a single-row write. Building it costs one + * Index traversal; reading it requires no Index traversal at all. + * + *

VersionedSQLTable stores its history {@code Index} at this same slot, so + * {@link #getBlockVec()} guards with {@code instanceof AVector} to distinguish. + */ + static final int POS_BLOCK_VEC = 4; + /** + * Position of secondary column indices. + * + *

Value is {@code Index>} mapping column name + * to a column index. The column index maps + * {@code encode(value) ++ pk_bytes} → pk_blob. + * May be {@code null} when no indices are defined. + */ + static final int POS_INDICES = 5; SQLTable(ALatticeCursor> cursor) { super(cursor); @@ -48,18 +77,19 @@ public class SQLTable extends ALatticeComponent> { * * @param schema Column definitions * @param timestamp Creation timestamp - * @return State vector [schema, empty-rows, timestamp, liveCount=0] + * @return State vector [schema, empty-rows, timestamp, liveCount=0, emptyBlockVec, emptyIndices] */ @SuppressWarnings("unchecked") static AVector createState(AVector> schema, CVMLong timestamp) { - return Vectors.of(schema, (Index>) Index.EMPTY, timestamp, CVMLong.ZERO); + return Vectors.of(schema, (Index) Index.EMPTY, timestamp, CVMLong.ZERO, + Vectors.empty(), Index.EMPTY); } /** * Creates a tombstone state vector for a dropped table. * * @param timestamp Deletion timestamp - * @return Tombstone state vector [null, null, timestamp] + * @return Tombstone state vector [null, null, timestamp, liveCount=0] */ static AVector createTombstoneState(CVMLong timestamp) { return Vectors.of(null, null, timestamp, CVMLong.ZERO); @@ -74,6 +104,151 @@ static boolean isLiveState(AVector state) { return state != null && state.get(POS_SCHEMA) != null; } + // ========== Static Helpers (block vector) ========== + + /** + * Builds a sequential block vector from a rows Index. + * Used to populate slot 4 for O(1)-heap full scans. + */ + @SuppressWarnings("unchecked") + static AVector buildBlockVec(Index rows) { + if (rows == null || rows.count() == 0) return Vectors.empty(); + List blocks = new ArrayList<>((int) Math.min(rows.count(), Integer.MAX_VALUE)); + rows.forEach((bk, block) -> { + if (RowBlock.isBlock(block)) blocks.add(block); + }); + return Vectors.create(blocks); + } + + // ========== Static Helpers (column indices) ========== + + /** + * Extracts the secondary column indices map from a raw state vector. + * Returns null if not present or empty. + */ + @SuppressWarnings("unchecked") + static Index> getIndicesFromState(AVector state) { + if (state == null || state.count() <= POS_INDICES) return null; + ACell slot = state.get(POS_INDICES); + if (!(slot instanceof Index)) return null; + Index> idx = (Index>) slot; + return (idx.count() == 0) ? null : idx; + } + + /** + * Finds the column position index within a schema vector by column name. + * Returns -1 if not found. + */ + static int findColIdxInSchema(AVector> schema, AString colName) { + if (schema == null) return -1; + for (int i = 0; i < (int) schema.count(); i++) { + if (colName.equals(schema.get(i).get(0))) return i; + } + return -1; + } + + /** + * Builds a fresh column index from all live rows in the given state. + * Used when {@link #createColumnIndex} is called on a table with existing data. + */ + @SuppressWarnings("unchecked") + static Index buildColumnIndex(AVector state, int colIdx) { + Index rows = (Index) state.get(POS_ROWS); + if (rows == null) return Index.none(); + + @SuppressWarnings("rawtypes") + Index[] result = {Index.none()}; + rows.forEach((bk, block) -> { + if (RowBlock.isBlock(block)) { + RowBlock.forEach(block, (pk, row) -> { + if (SQLRow.isLive(row)) { + AVector values = SQLRow.getValues(row); + if (colIdx < (int) values.count() && values.get(colIdx) != null) { + ABlob iKey = ColumnIndex.indexKey(values.get(colIdx), pk); + result[0] = result[0].assoc(iKey, pk); + } + } + }); + } + }); + return (Index) result[0]; + } + + /** + * Returns a copy of the indices map with a new or updated entry for the given row insert. + * Handles both new rows and updates to existing rows (removes old index entry first). + * + * @param indices Current indices map (may be null) + * @param schema Table schema (for column position lookup) + * @param pk Primary key of the row being inserted + * @param oldValues Old column values (null if this is a new row, not an update) + * @param newValues New column values + */ + @SuppressWarnings("unchecked") + static Index> indexAddRow( + Index> indices, + AVector> schema, + ABlob pk, AVector oldValues, AVector newValues) { + if (indices == null || indices.count() == 0) return indices; + + @SuppressWarnings("rawtypes") + final Index[] result = {indices}; + indices.forEach((colName, colIndex) -> { + int ci = findColIdxInSchema(schema, (AString) colName); + if (ci < 0) return; + + Index col = (Index) colIndex; + + // Remove old entry (if updating an existing live row) + if (oldValues != null && ci < (int) oldValues.count()) { + ACell oldVal = oldValues.get(ci); + if (oldVal != null) col = col.dissoc(ColumnIndex.indexKey(oldVal, pk)); + } + // Add new entry + if (ci < (int) newValues.count()) { + ACell newVal = newValues.get(ci); + if (newVal != null) col = col.assoc(ColumnIndex.indexKey(newVal, pk), pk); + } + + result[0] = result[0].assoc(colName, col); + }); + return (Index>) result[0]; + } + + /** + * Returns a copy of the indices map with the given row removed (delete/tombstone). + */ + @SuppressWarnings("unchecked") + static Index> indexRemoveRow( + Index> indices, + AVector> schema, + ABlob pk, AVector values) { + if (indices == null || indices.count() == 0) return indices; + + @SuppressWarnings("rawtypes") + final Index[] result = {indices}; + indices.forEach((colName, colIndex) -> { + int ci = findColIdxInSchema(schema, (AString) colName); + if (ci < 0 || ci >= (int) values.count()) return; + ACell val = values.get(ci); + if (val == null) return; + Index col = (Index) colIndex; + col = col.dissoc(ColumnIndex.indexKey(val, pk)); + result[0] = result[0].assoc(colName, col); + }); + return (Index>) result[0]; + } + + /** + * Extends or updates the indices slot (slot 5) in a state vector. + */ + private static AVector withIndices(AVector state, + ACell blockVec, CVMLong liveCount, + Index> indices, + ACell rows, ACell timestamp) { + return Vectors.of(state.get(POS_SCHEMA), rows, timestamp, liveCount, blockVec, indices); + } + // ========== Cursor-backed Instance Methods ========== /** @@ -96,15 +271,15 @@ public AVector> getSchema() { } /** - * Gets the rows from this table. + * Gets the rows index from this table. * - * @return Row index, or null if tombstone + * @return Row block index (ABlob prefix → RowBlock), or null if tombstone */ @SuppressWarnings("unchecked") - public Index> getRows() { + public Index getRows() { AVector state = cursor.get(); if (state == null) return null; - return (Index>) state.get(POS_ROWS); + return (Index) state.get(POS_ROWS); } /** @@ -116,6 +291,19 @@ public CVMLong getTimestamp() { return (CVMLong) state.get(POS_UTIME); } + /** + * Gets the sequential block vector (slot 4) for fast full scans. + * Returns null if not present, invalidated (null slot), or if slot 4 holds + * a non-AVector value (e.g. VersionedSQLTable's history Index). + */ + @SuppressWarnings("unchecked") + public AVector getBlockVec() { + AVector state = cursor.get(); + if (state == null || state.count() <= POS_BLOCK_VEC) return null; + ACell slot = state.get(POS_BLOCK_VEC); + return (slot instanceof AVector) ? (AVector) slot : null; + } + /** * Checks if this table is a tombstone (dropped). */ @@ -160,18 +348,99 @@ public long getColumnCount() { /** * Gets the number of live (non-tombstone) rows. O(1). */ + @SuppressWarnings("unchecked") public long getRowCount() { AVector state = cursor.get(); if (state == null) return 0; if (state.count() <= POS_LIVE_COUNT) { // Legacy state vector without liveCount — fall back to Index.count() - Index> rows = getRows(); + Index rows = getRows(); return (rows != null) ? rows.count() : 0; } CVMLong lc = (CVMLong) state.get(POS_LIVE_COUNT); return (lc != null) ? lc.longValue() : 0; } + // ========== Secondary Index Methods ========== + + /** + * Returns true if a secondary index exists on the named column. + */ + public boolean hasColumnIndex(AString colName) { + Index> indices = getIndicesFromState(cursor.get()); + return indices != null && indices.get(colName) != null; + } + + /** + * Creates a secondary index on the named column. + * Builds the index from existing rows. Returns false if the column doesn't exist + * or the index already exists. + */ + @SuppressWarnings("unchecked") + public boolean createColumnIndex(AString colName) { + boolean[] created = {false}; + cursor.updateAndGet(state -> { + if (state == null || state.get(POS_SCHEMA) == null) return state; + AVector> schema = (AVector>) state.get(POS_SCHEMA); + int colIdx = findColIdxInSchema(schema, colName); + if (colIdx < 0) return state; // column not found + + ACell rawIndices = (state.count() > POS_INDICES) ? state.get(POS_INDICES) : null; + Index> indices = + (rawIndices instanceof Index) ? (Index>) rawIndices + : Index.none(); + if (indices.get(colName) != null) return state; // already exists + + // Build index from existing rows + Index colIndex = buildColumnIndex(state, colIdx); + indices = indices.assoc(colName, colIndex); + created[0] = true; + + // Extend state to 6 slots if needed + if (state.count() <= POS_INDICES) { + return state.append(indices); + } + return state.assoc(POS_INDICES, indices); + }); + return created[0]; + } + + /** + * Drops the secondary index on the named column. + * Returns false if no such index exists. + */ + @SuppressWarnings("unchecked") + public boolean dropColumnIndex(AString colName) { + boolean[] dropped = {false}; + cursor.updateAndGet(state -> { + if (state == null || state.count() <= POS_INDICES) return state; + ACell rawIndices = state.get(POS_INDICES); + if (!(rawIndices instanceof Index)) return state; + Index> indices = + (Index>) rawIndices; + if (indices.get(colName) == null) return state; // doesn't exist + indices = indices.dissoc(colName); + dropped[0] = true; + return state.assoc(POS_INDICES, indices); + }); + return dropped[0]; + } + + /** + * Looks up a single row by its primary key blob (bypassing ACell→ABlob conversion). + * Returns the column values vector, or null if not found or tombstoned. + */ + @SuppressWarnings("unchecked") + public AVector selectByKeyBlob(ABlob pk) { + Index rows = getRows(); + if (rows == null) return null; + ABlob bk = RowBlock.blockKey(pk); + ACell block = rows.get(bk); + AVector row = RowBlock.get(block, pk); + if (row == null || !SQLRow.isLive(row)) return null; + return SQLRow.getValues(row); + } + // ========== Mutation Methods ========== /** @@ -187,19 +456,101 @@ public boolean insertRow(ABlob pk, AVector values, CVMLong timestamp) { boolean[] result = new boolean[1]; cursor.updateAndGet(state -> { if (state == null || state.get(POS_SCHEMA) == null) return state; - Index> rows = (Index>) state.get(POS_ROWS); - if (rows == null) rows = TableLattice.INSTANCE.zero(); - // Check if this insert adds a new live row - AVector existing = rows.get(pk); + Index rows = (Index) state.get(POS_ROWS); + if (rows == null) rows = BlockTableLattice.INSTANCE.zero(); + ABlob bk = RowBlock.blockKey(pk); + ACell block = rows.get(bk); + AVector existing = RowBlock.get(block, pk); boolean addsLive = (existing == null || !SQLRow.isLive(existing)); - rows = rows.assoc(pk, SQLRow.create(values, timestamp)); + AVector oldValues = (existing != null && SQLRow.isLive(existing)) + ? SQLRow.getValues(existing) : null; + ACell newBlock = RowBlock.put(block, pk, SQLRow.create(values, timestamp)); + rows = rows.assoc(bk, newBlock); long liveCount = getLiveCount(state) + (addsLive ? 1 : 0); result[0] = true; - return Vectors.of(state.get(POS_SCHEMA), rows, timestamp, CVMLong.create(liveCount)); + + // Update column indices (remove old entry if updating, add new) + AVector> schema = (AVector>) state.get(POS_SCHEMA); + Index> indices = + indexAddRow(getIndicesFromState(state), schema, pk, oldValues, values); + + return withIndices(state, null, CVMLong.create(liveCount), indices, rows, timestamp); }); return result[0]; } + /** + * Batch-inserts pre-sorted rows in a single atomic update. + * + *

Rows sharing the same block-key prefix are grouped into one block and + * committed with a single {@code Index.assoc()} call, reducing intermediate + * allocation from O(N × trie-depth) to O(blocks × trie-depth). + * + * @param sortedEntries (pk → row-values) pairs sorted by pk ascending + * @param timestamp Write timestamp + * @return number of newly-live rows inserted + */ + @SuppressWarnings("unchecked") + public int insertRows(List>> sortedEntries, CVMLong timestamp) { + if (sortedEntries.isEmpty()) return 0; + int[] newLive = {0}; + cursor.updateAndGet(state -> { + if (state == null || state.get(POS_SCHEMA) == null) return state; + Index rows = (Index) state.get(POS_ROWS); + if (rows == null) rows = BlockTableLattice.INSTANCE.zero(); + + // Group entries by block key, then use putAll for each block. + // For fresh (empty) tables: collect blocks inline to build blockVec at O(batch) + // cost. For non-empty tables: invalidate blockVec (avoids O(n) full-Index traversal). + boolean wasEmpty = getLiveCount(state) == 0; + ABlob curBk = null; + List blockPks = new ArrayList<>(); + List> blockRows = new ArrayList<>(); + List blockList = wasEmpty ? new ArrayList<>() : null; + + for (var e : sortedEntries) { + ABlob pk = e.getKey(); + ABlob bk = RowBlock.blockKey(pk); + if (curBk == null || !bk.equals(curBk)) { + if (curBk != null) { + ACell existing = rows.get(curBk); + ACell newBlock = RowBlock.putAll(existing, blockPks, blockRows, newLive); + rows = rows.assoc(curBk, newBlock); + if (blockList != null) blockList.add(newBlock); + blockPks = new ArrayList<>(); + blockRows = new ArrayList<>(); + } + curBk = bk; + } + blockPks.add(pk); + blockRows.add(SQLRow.create(e.getValue(), timestamp)); + } + if (curBk != null) { + ACell existing = rows.get(curBk); + ACell newBlock = RowBlock.putAll(existing, blockPks, blockRows, newLive); + rows = rows.assoc(curBk, newBlock); + if (blockList != null) blockList.add(newBlock); + } + + long liveCount = getLiveCount(state) + newLive[0]; + // blockVec: inline-built for fresh tables (O(batch)), null for incremental inserts + AVector blockVec = (blockList != null) ? Vectors.create(blockList) : null; + + // Update column indices for all inserted rows + AVector> schema = (AVector>) state.get(POS_SCHEMA); + Index> indices = getIndicesFromState(state); + if (indices != null && indices.count() > 0) { + for (var e : sortedEntries) { + // For batch inserts (typically fresh tables), no old-value removal needed + indices = indexAddRow(indices, schema, e.getKey(), null, e.getValue()); + } + } + + return withIndices(state, blockVec, CVMLong.create(liveCount), indices, rows, timestamp); + }); + return newLive[0]; + } + /** * Deletes a row by primary key. * @@ -212,14 +563,25 @@ public boolean deleteRow(ABlob key, CVMLong timestamp) { boolean[] result = new boolean[1]; cursor.updateAndGet(state -> { if (state == null || state.get(POS_SCHEMA) == null) return state; - Index> rows = (Index>) state.get(POS_ROWS); + Index rows = (Index) state.get(POS_ROWS); if (rows == null) return state; - AVector row = rows.get(key); - if (row == null || !SQLRow.isLive(row)) return state; - rows = rows.assoc(key, SQLRow.createTombstone(timestamp)); + ABlob bk = RowBlock.blockKey(key); + ACell block = rows.get(bk); + AVector existing = RowBlock.get(block, key); + if (existing == null || !SQLRow.isLive(existing)) return state; + AVector oldValues = SQLRow.getValues(existing); + ACell newBlock = RowBlock.put(block, key, SQLRow.createTombstone(timestamp)); + rows = rows.assoc(bk, newBlock); long liveCount = getLiveCount(state) - 1; result[0] = true; - return Vectors.of(state.get(POS_SCHEMA), rows, timestamp, CVMLong.create(liveCount)); + + // Remove from column indices + AVector> schema = (AVector>) state.get(POS_SCHEMA); + Index> indices = + indexRemoveRow(getIndicesFromState(state), schema, key, oldValues); + + // Invalidate blockVec (tombstone inserted; scan must skip it) + return withIndices(state, null, CVMLong.create(liveCount), indices, rows, timestamp); }); return result[0]; } @@ -235,24 +597,72 @@ static long getLiveCount(AVector state) { return (lc != null) ? lc.longValue() : 0; } + /** + * Extracts blockVec from a raw state vector, or null if absent/invalid. + */ + @SuppressWarnings("unchecked") + static AVector getBlockVecFromState(AVector state) { + if (state == null || state.count() <= POS_BLOCK_VEC) return null; + ACell slot = state.get(POS_BLOCK_VEC); + return (slot instanceof AVector) ? (AVector) slot : null; + } + /** * Computes live row count by scanning an Index. Used only during merge. */ - static long computeLiveCount(Index> rows) { + @SuppressWarnings("unchecked") + static long computeLiveCount(Index rows) { if (rows == null) return 0; long[] count = new long[1]; - rows.forEach((k, v) -> { - if (SQLRow.isLive(v)) count[0]++; + rows.forEach((bk, block) -> { + if (RowBlock.isBlock(block)) { + RowBlock.forEach(block, (pk, row) -> { if (SQLRow.isLive(row)) count[0]++; }); + } else if (block instanceof AVector && SQLRow.isLive((AVector)block)) { + // Legacy single raw row entry (backward compat) + count[0]++; + } }); return count[0]; } + /** + * Merges two column indices maps using union semantics. + * Entries from both sides are unioned. Stale entries (for deleted rows) are + * filtered at query time by re-validating against the row store. + */ + @SuppressWarnings("unchecked") + static Index> mergeColumnIndices( + Index> a, + Index> b) { + if (a == null || a.count() == 0) return b; + if (b == null || b.count() == 0) return a; + + @SuppressWarnings({"unchecked", "rawtypes"}) + final Index[] result = {a}; + b.forEach((colName, bColIdx) -> { + ACell rawA = ((Index) result[0]).get(colName); + if (!(rawA instanceof Index)) { + // a doesn't have this column index — use b's + result[0] = result[0].assoc(colName, bColIdx); + } else { + // Both have it — union merge: assoc all of b's entries into a's + @SuppressWarnings("rawtypes") + final Index[] mergedCol = {(Index) rawA}; + ((Index) bColIdx).forEach((k, v) -> + mergedCol[0] = mergedCol[0].assoc(k, v)); + result[0] = result[0].assoc(colName, mergedCol[0]); + } + }); + return (Index>) result[0]; + } + // ========== Static Merge (used by lattice layer) ========== /** * Merges two table state vectors. * Schema uses LWW (latest timestamp wins). - * Rows merge using TableLattice. + * Rows merge using BlockTableLattice. + * Column indices use union merge. */ @SuppressWarnings("unchecked") public static AVector merge(AVector a, AVector b) { @@ -282,22 +692,34 @@ public static AVector merge(AVector a, AVector b) { } // Merge rows - Index> rowsA = (Index>) a.get(POS_ROWS); - Index> rowsB = (Index>) b.get(POS_ROWS); - Index> mergedRows = TableLattice.INSTANCE.merge(rowsA, rowsB); + Index rowsA = (Index) a.get(POS_ROWS); + Index rowsB = (Index) b.get(POS_ROWS); + Index mergedRows = BlockTableLattice.INSTANCE.merge(rowsA, rowsB); - // Compute live count for merged result + // Compute live count and blockVec for merged result long liveCount; + AVector blockVec; + Index> mergedIndices; if (mergedRows == rowsA) { liveCount = getLiveCount(a); + blockVec = getBlockVecFromState(a); + mergedIndices = getIndicesFromState(a); } else if (mergedRows == rowsB) { liveCount = getLiveCount(b); + blockVec = getBlockVecFromState(b); + mergedIndices = getIndicesFromState(b); } else { - // Rows actually merged — recompute + // Rows actually merged — recompute liveCount; invalidate blockVec; + // union-merge column indices (stale entries filtered at query time) liveCount = computeLiveCount(mergedRows); + blockVec = null; + mergedIndices = mergeColumnIndices( + getIndicesFromState(a), getIndicesFromState(b)); } // Return merged table with schema winner's schema - return Vectors.of(schemaWinner.get(POS_SCHEMA), mergedRows, schemaWinner.get(POS_UTIME), CVMLong.create(liveCount)); + return Vectors.of(schemaWinner.get(POS_SCHEMA), mergedRows, + schemaWinner.get(POS_UTIME), CVMLong.create(liveCount), + blockVec, mergedIndices); } } diff --git a/convex-db/src/main/java/convex/db/lattice/SQLTableLattice.java b/convex-db/src/main/java/convex/db/lattice/SQLTableLattice.java index 1fe20b102..52a91b39d 100644 --- a/convex-db/src/main/java/convex/db/lattice/SQLTableLattice.java +++ b/convex-db/src/main/java/convex/db/lattice/SQLTableLattice.java @@ -37,7 +37,7 @@ public ALattice path(ACell childKey) { // Table vector: [0]=schema, [1]=rows, [2]=utime // Only rows (position 1) has a sub-lattice if (childKey instanceof CVMLong idx && idx.longValue() == SQLTable.POS_ROWS) { - return (ALattice) TableLattice.INSTANCE; + return (ALattice) BlockTableLattice.INSTANCE; } return null; } diff --git a/convex-db/src/main/java/convex/db/lattice/VersionedSQLSchema.java b/convex-db/src/main/java/convex/db/lattice/VersionedSQLSchema.java new file mode 100644 index 000000000..681730b29 --- /dev/null +++ b/convex-db/src/main/java/convex/db/lattice/VersionedSQLSchema.java @@ -0,0 +1,256 @@ +package convex.db.lattice; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import convex.core.data.ABlob; +import convex.core.data.ACell; +import convex.core.data.AString; +import convex.core.data.AVector; +import convex.core.data.Index; +import convex.core.data.Strings; +import convex.core.data.Vectors; +import convex.core.data.prim.CVMLong; +import convex.db.calcite.ConvexColumnType; +import convex.db.calcite.ConvexType; +import convex.lattice.cursor.ALatticeCursor; +import convex.lattice.cursor.Cursors; + +/** + * System-versioned SQL schema — a {@link SQLSchema} that records every insert, + * update, and deletion in a per-table history index. + * + *

Extends {@link SQLSchema} with: + *

    + *
  • All table state vectors include a 5th slot: {@code history} (see {@link VersionedSQLTable#POS_HISTORY})
    • + *
  • {@link #insert} and {@link #deleteByKey} automatically append history entries
    • + *
  • Deduplication: writes with identical content are silently skipped
    • + *
  • {@link #getHistory} — all change events for a primary key, oldest first
    • + *
  • {@link #getAsOf} — row state at an arbitrary point in time
    • + *
+ * + *

Lattice merge uses {@link VersionedTableStoreLattice}, which union-merges the + * history slot across replicas — old versions survive replication and are never lost. + * + *

Timestamps: + *

    + *
  • History keys use {@link System#nanoTime()} — monotonic within a JVM run, precise ordering
    • + *
  • LWW conflict-resolution uses {@link System#currentTimeMillis()} — comparable across nodes
    • + *
+ * + *

SQL extension note: Calcite {@code AS OF SYSTEM TIME} is not yet wired up. + * {@link #getHistory} and {@link #getAsOf} provide equivalent functionality via the lattice API. + */ +public class VersionedSQLSchema extends SQLSchema { + + VersionedSQLSchema(ALatticeCursor>> cursor) { + super(cursor); + } + + /** + * Creates a new standalone versioned schema backed by an in-memory cursor. + * + * @return New VersionedSQLSchema instance + */ + public static VersionedSQLSchema create() { + ALatticeCursor>> cursor = + Cursors.createLattice(VersionedTableStoreLattice.INSTANCE); + return new VersionedSQLSchema(cursor); + } + + /** + * Connects to an existing cursor (e.g. from a NodeServer chain) for persistence. + * + * @param cursor Lattice cursor pointing at the table store + * @return New VersionedSQLSchema instance + */ + public static VersionedSQLSchema connect(ALatticeCursor>> cursor) { + return new VersionedSQLSchema(cursor); + } + + /** + * Wraps an existing {@link SQLSchema} with versioned table support, sharing the + * same underlying cursor. Useful when the schema was created via + * {@link SQLDatabase#tables()} and you want history tracking without changing + * the cursor chain setup. + * + *

Note: the underlying lattice for replication merges remains the original + * (non-versioned) one. History is stored correctly in the cursor state but may + * not survive cross-node merge until the NodeServer is configured with + * {@link VersionedTableStoreLattice}. For single-node use this is not an issue. + * + * @param schema Existing SQLSchema instance + * @return VersionedSQLSchema sharing the same cursor + */ + public static VersionedSQLSchema wrap(SQLSchema schema) { + return new VersionedSQLSchema(schema.cursor()); + } + + // ── Table accessors (return VersionedSQLTable) ─────────────────────────── + + @Override + public VersionedSQLTable getTable(AString name) { + ALatticeCursor> tableCursor = cursor.path(name); + if (tableCursor.get() == null) return null; + return new VersionedSQLTable(tableCursor); + } + + @Override + public VersionedSQLTable getTable(String name) { + return getTable(Strings.create(name)); + } + + @Override + public VersionedSQLTable getLiveTable(AString name) { + VersionedSQLTable table = getTable(name); + if (table == null || !table.isLive()) return null; + return table; + } + + @Override + public VersionedSQLTable getLiveTable(String name) { + return getLiveTable(Strings.create(name)); + } + + // ── Table creation (5-slot state vector) ──────────────────────────────── + + /** + * Creates a versioned table. The state vector includes an empty history slot. + * All other {@code createTable} overloads delegate here through the parent class. + */ + @Override + @SuppressWarnings({"unchecked", "rawtypes"}) + public boolean createTable(AString name, String[] columns, ConvexColumnType[] types) { + if (columns.length != types.length) + throw new IllegalArgumentException("Columns and types must have same length"); + + AVector newSchema = buildSchema(columns, types); + + VersionedSQLTable existing = getLiveTable(name); + if (existing == null) { + ALatticeCursor> tableCursor = cursor.path(name); + tableCursor.set(VersionedSQLTable.createState((AVector>) newSchema, millis())); + return true; + } + + // Table already exists: append any new columns not present in stored schema + AVector> existingSchema = existing.getSchema(); + long existingCount = existingSchema != null ? existingSchema.count() : 0; + if (newSchema.count() <= existingCount) return false; + + AVector combinedSchema = existingSchema; + for (long i = existingCount; i < newSchema.count(); i++) { + combinedSchema = combinedSchema.append(newSchema.get(i)); + } + final AVector> finalSchema = (AVector>) combinedSchema; + ALatticeCursor> tableCursor = cursor.path(name); + tableCursor.updateAndGet(state -> state.assoc(SQLTable.POS_SCHEMA, finalSchema)); + return true; + } + + // ── Versioned mutations ────────────────────────────────────────────────── + + /** + * Inserts or updates a row, recording the change in the history index. + * Skips if values are identical to the current live row (deduplication). + */ + @Override + public boolean insert(AString tableName, AVector row) { + VersionedSQLTable table = getLiveTable(tableName); + if (table == null) return false; + ABlob pk = toKey(row.get(0)); + return table.insertRowVersioned(pk, row, System.nanoTime(), millis()); + } + + /** + * Marks a row as deleted, recording the deletion in the history index. + */ + @Override + public boolean deleteByKey(AString tableName, ACell primaryKey) { + VersionedSQLTable table = getLiveTable(tableName); + if (table == null) return false; + ABlob pk = toKey(primaryKey); + return table.deleteRowVersioned(pk, System.nanoTime(), millis()); + } + + // ── Temporal query API ─────────────────────────────────────────────────── + + /** + * Returns all recorded change events for a primary key, oldest first. + * Each entry is {@code [values|null, CVMLong(nanotime), CVMLong(changeType)]}. + * + * @param tableName Table name + * @param primaryKey Primary key value (same types as insert) + * @return Ordered list of history entries; empty if table not found or no history + */ + public List> getHistory(String tableName, ACell primaryKey) { + return getHistory(Strings.create(tableName), primaryKey); + } + + public List> getHistory(AString tableName, ACell primaryKey) { + VersionedSQLTable table = getLiveTable(tableName); + if (table == null) return List.of(); + return table.getHistory(toKey(primaryKey)); + } + + /** + * Returns the row as it existed at or before the given nanotime + * (equivalent to {@code SELECT ... AS OF SYSTEM TIME}). + * + * @param tableName Table name + * @param primaryKey Primary key value + * @param nanotime Upper-bound nanotime (from {@link System#nanoTime()} at the capture point) + * @return History entry at that point, or null if no version existed + */ + public AVector getAsOf(String tableName, ACell primaryKey, long nanotime) { + return getAsOf(Strings.create(tableName), primaryKey, nanotime); + } + + public AVector getAsOf(AString tableName, ACell primaryKey, long nanotime) { + VersionedSQLTable table = getLiveTable(tableName); + if (table == null) return null; + return table.getAsOf(toKey(primaryKey), nanotime); + } + + // ── Batch insert ──────────────────────────────────────────────────────── + + /** + * Batch-inserts rows with history tracking in a single atomic update. + * Overrides {@link SQLSchema#insertAll} to include per-row history entries. + */ + @Override + public int insertAll(AString tableName, List> rows) { + if (rows == null || rows.isEmpty()) return 0; + VersionedSQLTable table = getLiveTable(tableName); + if (table == null) return 0; + long startNano = System.nanoTime(); + CVMLong millis = millis(); + List>> sorted = new ArrayList<>(rows.size()); + for (AVector row : rows) { + sorted.add(Map.entry(toKey(row.get(0)), row)); + } + sorted.sort(Map.Entry.comparingByKey()); + return table.insertRowsVersioned(sorted, startNano, millis); + } + + // ── Helpers ────────────────────────────────────────────────────────────── + + private CVMLong millis() { + return CVMLong.create(System.currentTimeMillis()); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private static AVector buildSchema(String[] columns, ConvexColumnType[] types) { + AVector schema = Vectors.empty(); + for (int i = 0; i < columns.length; i++) { + ConvexColumnType ct = types[i]; + CVMLong precision = ct.hasPrecision() ? CVMLong.create(ct.getPrecision()) : null; + CVMLong scale = ct.hasScale() ? CVMLong.create(ct.getScale()) : null; + convex.core.data.AString typeName = + (ct.getBaseType() == ConvexType.ANY) ? null : Strings.create(ct.getBaseType().name()); + schema = schema.append(Vectors.of(Strings.create(columns[i]), typeName, precision, scale)); + } + return schema; + } +} diff --git a/convex-db/src/main/java/convex/db/lattice/VersionedSQLTable.java b/convex-db/src/main/java/convex/db/lattice/VersionedSQLTable.java new file mode 100644 index 000000000..f5dbae03c --- /dev/null +++ b/convex-db/src/main/java/convex/db/lattice/VersionedSQLTable.java @@ -0,0 +1,368 @@ +package convex.db.lattice; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import convex.core.data.ABlob; +import convex.core.data.ACell; +import convex.core.data.AVector; +import convex.core.data.Blob; +import convex.core.data.Index; +import convex.core.data.Vectors; +import convex.core.data.prim.CVMLong; +import convex.lattice.ALattice; +import convex.lattice.cursor.ALatticeCursor; +import convex.lattice.generic.IndexLattice; + +/** + * A system-versioned SQL table: extends {@link SQLTable} with a history index + * that records every insert, update, and deletion with a nanotime timestamp. + * + *

State vector: {@code [schema, rows, utime, liveCount, history]} + *

    + *
  • {@code history} — {@code Index>} keyed by + * {@link HistoryKey} (pk ++ nanotime), value = {@code [Blob(values)|null, nanotime, changeType]}; + * use {@link #getHistoryValues} to decode the values slot
    • + *
+ * + *

Change types: + *

    + *
  • {@link #CT_INSERT} = 1
    • + *
  • {@link #CT_UPDATE} = 2
    • + *
  • {@link #CT_DELETE} = 3
    • + *
+ * + *

Deduplication: writes are skipped if the incoming values are identical to the + * current live row (same content → same Etch cell hash). + * + *

Merge: history uses union semantics — history keys are unique by design + * (pk + nanotime), so merging two replicas simply takes all entries from both. + */ +public class VersionedSQLTable extends SQLTable { + + /** Position of the history index in the versioned state vector. */ + static final int POS_HISTORY = 4; + + /** Change type: row was inserted (no prior live row for this pk). */ + public static final long CT_INSERT = 1; + /** Change type: row was updated (prior live row existed). */ + public static final long CT_UPDATE = 2; + /** Change type: row was deleted. */ + public static final long CT_DELETE = 3; + + /** + * Leaf lattice for individual history entries. + * History keys are unique by (pk, nanotime), so conflicts do not occur in practice. + * When they do (same nanotime), own value wins per lattice convention. + */ + @SuppressWarnings("unchecked") + static final ALattice> HISTORY_ENTRY_LATTICE = new ALattice<>() { + @Override public AVector zero() { return null; } + @Override public AVector merge(AVector a, AVector b) { return a != null ? a : b; } + @Override public boolean checkForeign(AVector v) { return v instanceof AVector; } + @Override public ALattice path(ACell k) { return null; } + }; + + /** IndexLattice for the history slot — union merge via the identity entry lattice. */ + static final IndexLattice> HISTORY_LATTICE = + IndexLattice.create(HISTORY_ENTRY_LATTICE); + + VersionedSQLTable(ALatticeCursor> cursor) { + super(cursor); + } + + // ── State factories ────────────────────────────────────────────────────── + + /** + * Creates the initial 5-slot state vector for a new versioned table. + */ + @SuppressWarnings("unchecked") + static AVector createState(AVector> schema, CVMLong timestamp) { + return Vectors.of( + schema, + (Index) Index.EMPTY, // rows + timestamp, + CVMLong.ZERO, // liveCount + (Index>) Index.EMPTY // history + ); + } + + // ── History accessor ───────────────────────────────────────────────────── + + /** Returns the raw history index from the current cursor state. */ + @SuppressWarnings("unchecked") + public Index> getHistoryIndex() { + return historyFrom(getState()); + } + + // ── Versioned mutations ────────────────────────────────────────────────── + + /** + * Inserts or updates a row, also appending a history entry. + * No-op (returns false) if values are identical to the current live row. + * + * @param pk Primary key blob + * @param values Full row values vector + * @param nanotime Monotonic timestamp for history ordering + * @param milliTimestamp Wall-clock timestamp for LWW conflict resolution + * @return true if the row was written; false if skipped as duplicate + */ + @SuppressWarnings("unchecked") + public boolean insertRowVersioned(ABlob pk, AVector values, long nanotime, CVMLong milliTimestamp) { + boolean[] changed = {false}; + cursor.updateAndGet(state -> { + if (!isLiveState(state)) return state; + + Index rows = rowsFrom(state); + ABlob bk = RowBlock.blockKey(pk); + ACell block = rows.get(bk); + AVector existing = RowBlock.get(block, pk); + + // Deduplication: identical content is a no-op + if (existing != null && SQLRow.isLive(existing) && values.equals(SQLRow.getValues(existing))) + return state; + + long changeType = (existing == null || !SQLRow.isLive(existing)) ? CT_INSERT : CT_UPDATE; + boolean addsLive = (existing == null || !SQLRow.isLive(existing)); + + rows = rows.assoc(bk, RowBlock.put(block, pk, SQLRow.create(values, milliTimestamp))); + long liveCount = getLiveCount(state) + (addsLive ? 1 : 0); + + Index> history = historyFrom(state); + history = history.assoc( + HistoryKey.of(pk, nanotime), + Vectors.of(SQLRow.encodeValues(values), CVMLong.create(nanotime), CVMLong.create(changeType)) + ); + + changed[0] = true; + return buildState(state.get(POS_SCHEMA), rows, milliTimestamp, CVMLong.create(liveCount), history); + }); + return changed[0]; + } + + /** + * Batch-inserts pre-sorted rows with history tracking in a single atomic update. + * + *

Applies the same block-grouping optimisation as {@link SQLTable#insertRows} + * and additionally records one history entry per non-deduplicated row. + * Nanotimes are assigned sequentially from {@code startNanotime} to guarantee + * ordering within the batch. + * + * @param sortedEntries (pk → row-values) pairs sorted by pk ascending + * @param startNanotime Monotonic base timestamp; incremented per row written + * @param milliTimestamp Wall-clock timestamp for LWW conflict resolution + * @return number of newly-live rows inserted + */ + @SuppressWarnings("unchecked") + public int insertRowsVersioned(List>> sortedEntries, + long startNanotime, CVMLong milliTimestamp) { + if (sortedEntries.isEmpty()) return 0; + int[] newLive = {0}; + cursor.updateAndGet(state -> { + if (!isLiveState(state)) return state; + Index rows = rowsFrom(state); + Index> history = historyFrom(state); + + ABlob curBk = null; + List blockPks = new ArrayList<>(); + List> blockRows = new ArrayList<>(); + // We also need to track history entries per block group + // but history entries are individual per row — collect them separately + long nanotime = startNanotime; + + // We need to handle deduplication check before batching + // so we use the single-row path for history but batch for blocks + // Re-implement with block-grouped putAll for rows, individual history entries + for (var e : sortedEntries) { + ABlob pk = e.getKey(); + AVector values = e.getValue(); + ABlob bk = RowBlock.blockKey(pk); + + if (curBk == null || !bk.equals(curBk)) { + if (curBk != null) { + ACell existing = rows.get(curBk); + ACell newBlock = RowBlock.putAll(existing, blockPks, blockRows, null); + rows = rows.assoc(curBk, newBlock); + blockPks = new ArrayList<>(); + blockRows = new ArrayList<>(); + } + curBk = bk; + } + + // Check existing for deduplication and live-count tracking + // We need to look up in the current rows index (before this batch) + // For simplicity with dedup: read from rows as currently modified + // (We flush each block group before moving to the next, so rows is up to date for prior groups) + ACell curBlock = rows.get(bk); + // Also account for already-queued entries in blockPks for this block + AVector existing = RowBlock.get(curBlock, pk); + // Check deduplication + if (existing != null && SQLRow.isLive(existing) + && values.equals(SQLRow.getValues(existing))) { + nanotime++; + continue; + } + + long changeType = (existing == null || !SQLRow.isLive(existing)) ? CT_INSERT : CT_UPDATE; + if (changeType == CT_INSERT) newLive[0]++; + + blockPks.add(pk); + blockRows.add(SQLRow.create(values, milliTimestamp)); + + history = history.assoc( + HistoryKey.of(pk, nanotime), + Vectors.of(SQLRow.encodeValues(values), CVMLong.create(nanotime), CVMLong.create(changeType)) + ); + nanotime++; + } + if (curBk != null && !blockPks.isEmpty()) { + ACell existing = rows.get(curBk); + ACell newBlock = RowBlock.putAll(existing, blockPks, blockRows, null); + rows = rows.assoc(curBk, newBlock); + } + + long liveCount = getLiveCount(state) + newLive[0]; + return buildState(state.get(POS_SCHEMA), rows, milliTimestamp, CVMLong.create(liveCount), history); + }); + return newLive[0]; + } + + /** + * Marks a row as deleted, also appending a history entry with {@link #CT_DELETE}. + * No-op if the row does not exist or is already deleted. + * + * @param pk Primary key blob + * @param nanotime Monotonic timestamp for history ordering + * @param milliTimestamp Wall-clock timestamp for LWW conflict resolution + * @return true if the row was deleted; false if not found + */ + @SuppressWarnings("unchecked") + public boolean deleteRowVersioned(ABlob pk, long nanotime, CVMLong milliTimestamp) { + boolean[] changed = {false}; + cursor.updateAndGet(state -> { + if (!isLiveState(state)) return state; + + Index rows = rowsFrom(state); + ABlob bk = RowBlock.blockKey(pk); + ACell block = rows.get(bk); + AVector existing = RowBlock.get(block, pk); + if (existing == null || !SQLRow.isLive(existing)) return state; + + rows = rows.assoc(bk, RowBlock.put(block, pk, SQLRow.createTombstone(milliTimestamp))); + long liveCount = getLiveCount(state) - 1; + + Index> history = historyFrom(state); + history = history.assoc( + HistoryKey.of(pk, nanotime), + Vectors.of(null, CVMLong.create(nanotime), CVMLong.create(CT_DELETE)) + ); + + changed[0] = true; + return buildState(state.get(POS_SCHEMA), rows, milliTimestamp, CVMLong.create(liveCount), history); + }); + return changed[0]; + } + + // ── History entry accessors ────────────────────────────────────────────── + + /** + * Gets the column values from a history entry. + * Handles v3 compact (Blob-encoded) and legacy (AVector) formats. + * Returns null for delete entries. + */ + @SuppressWarnings("unchecked") + public static AVector getHistoryValues(AVector histEntry) { + if (histEntry == null) return null; + ACell cell = histEntry.get(0); + if (cell == null) return null; // delete entry + if (cell instanceof Blob blob) return SQLRow.decodeValues(blob); // v3 compact + return (AVector) cell; // legacy + } + + // ── Temporal queries ───────────────────────────────────────────────────── + + /** + * Returns all history entries for the given pk, oldest first. + * Each entry is {@code [values|null, CVMLong(nanotime), CVMLong(changeType)]}. + * + * @param pk Primary key blob + * @return Ordered list of history entries; empty if none recorded + */ + public List> getHistory(ABlob pk) { + ABlob prefix = HistoryKey.prefix(pk); + List> result = new ArrayList<>(); + getHistoryIndex().forEach((k, v) -> { + if (HistoryKey.hasPrefix(k, prefix)) result.add(v); + }); + return result; + } + + /** + * Returns the latest history entry for pk at or before the given nanotime + * (i.e., the row as it existed at that point in time). + * + * @param pk Primary key blob + * @param nanotime Upper bound timestamp (inclusive) + * @return History entry, or null if no version existed at that point + */ + @SuppressWarnings("unchecked") + public AVector getAsOf(ABlob pk, long nanotime) { + ABlob prefix = HistoryKey.prefix(pk); + AVector[] best = new AVector[1]; + getHistoryIndex().forEach((k, v) -> { + if (HistoryKey.hasPrefix(k, prefix) && HistoryKey.extractNanotime(k) <= nanotime) + best[0] = v; // forEach is ordered → last match is the latest ≤ nanotime + }); + return best[0]; + } + + // ── Merge ──────────────────────────────────────────────────────────────── + + /** + * Merges two versioned table state vectors. + * Delegates schema + rows to {@link SQLTable#merge}, then union-merges history. + */ + @SuppressWarnings("unchecked") + public static AVector merge(AVector a, AVector b) { + AVector base = SQLTable.merge(a, b); + if (base == null) return null; + + Index> histA = historyFrom(a); + Index> histB = historyFrom(b); + Index> merged = HISTORY_LATTICE.merge(histA, histB); + + // If history is unchanged from the base winner, no new vector needed + if (base.count() > POS_HISTORY && base.get(POS_HISTORY) == merged) return base; + + return buildState( + base.get(POS_SCHEMA), + (Index) base.get(POS_ROWS), + (CVMLong) base.get(POS_UTIME), + (CVMLong) base.get(POS_LIVE_COUNT), + merged + ); + } + + // ── Private helpers ────────────────────────────────────────────────────── + + @SuppressWarnings("unchecked") + static Index rowsFrom(AVector state) { + if (state == null) return (Index) Index.EMPTY; + ACell r = state.get(POS_ROWS); + return r != null ? (Index) r : (Index) Index.EMPTY; + } + + @SuppressWarnings("unchecked") + static Index> historyFrom(AVector state) { + if (state == null || state.count() <= POS_HISTORY) return (Index>) Index.EMPTY; + ACell h = state.get(POS_HISTORY); + return h != null ? (Index>) h : (Index>) Index.EMPTY; + } + + static AVector buildState(ACell schema, + Index rows, CVMLong utime, + CVMLong liveCount, Index> history) { + return Vectors.of(schema, rows, utime, liveCount, history); + } +} diff --git a/convex-db/src/main/java/convex/db/lattice/VersionedSQLTableLattice.java b/convex-db/src/main/java/convex/db/lattice/VersionedSQLTableLattice.java new file mode 100644 index 000000000..f3f728d79 --- /dev/null +++ b/convex-db/src/main/java/convex/db/lattice/VersionedSQLTableLattice.java @@ -0,0 +1,47 @@ +package convex.db.lattice; + +import convex.core.data.ABlob; +import convex.core.data.ACell; +import convex.core.data.AVector; +import convex.core.data.prim.CVMLong; +import convex.lattice.ALattice; + +/** + * Lattice for a single versioned SQL table entry. + * + *

Extends the behaviour of {@link SQLTableLattice} by: + *

    + *
  • Delegating merge to {@link VersionedSQLTable#merge} (which union-merges the history slot)
    • + *
  • Exposing {@link VersionedSQLTable#HISTORY_LATTICE} at path position {@link VersionedSQLTable#POS_HISTORY}
    • + *
+ */ +public class VersionedSQLTableLattice extends ALattice> { + + public static final VersionedSQLTableLattice INSTANCE = new VersionedSQLTableLattice(); + + private VersionedSQLTableLattice() {} + + @Override + public AVector zero() { return null; } + + @Override + public AVector merge(AVector a, AVector b) { + return VersionedSQLTable.merge(a, b); + } + + @Override + public boolean checkForeign(AVector value) { + return value instanceof AVector && value.count() >= 5; + } + + @SuppressWarnings("unchecked") + @Override + public ALattice path(ACell childKey) { + if (childKey instanceof CVMLong idx) { + long pos = idx.longValue(); + if (pos == SQLTable.POS_ROWS) return (ALattice) BlockTableLattice.INSTANCE; + if (pos == VersionedSQLTable.POS_HISTORY) return (ALattice) VersionedSQLTable.HISTORY_LATTICE; + } + return null; + } +} diff --git a/convex-db/src/main/java/convex/db/lattice/VersionedTableStoreLattice.java b/convex-db/src/main/java/convex/db/lattice/VersionedTableStoreLattice.java new file mode 100644 index 000000000..8fe3c6f40 --- /dev/null +++ b/convex-db/src/main/java/convex/db/lattice/VersionedTableStoreLattice.java @@ -0,0 +1,49 @@ +package convex.db.lattice; + +import convex.core.data.ACell; +import convex.core.data.AString; +import convex.core.data.AVector; +import convex.core.data.Index; +import convex.lattice.ALattice; +import convex.lattice.generic.IndexLattice; + +/** + * Top-level lattice for a versioned table store. + * + *

Mirrors {@link TableStoreLattice} but uses {@link VersionedSQLTableLattice} + * for per-table merge, so the history slot is union-merged on replication. + */ +public class VersionedTableStoreLattice extends ALattice>> { + + public static final VersionedTableStoreLattice INSTANCE = new VersionedTableStoreLattice(); + + private final IndexLattice> delegate; + + private VersionedTableStoreLattice() { + this.delegate = IndexLattice.create(VersionedSQLTableLattice.INSTANCE); + } + + @Override + @SuppressWarnings("unchecked") + public Index> zero() { + return (Index>) Index.EMPTY; + } + + @Override + public Index> merge( + Index> a, + Index> b) { + return delegate.merge(a, b); + } + + @Override + public boolean checkForeign(Index> value) { + return delegate.checkForeign(value); + } + + @SuppressWarnings("unchecked") + @Override + public ALattice path(ACell childKey) { + return delegate.path(childKey); + } +} diff --git a/convex-db/src/main/java/convex/db/store/SegmentedEtchStore.java b/convex-db/src/main/java/convex/db/store/SegmentedEtchStore.java new file mode 100644 index 000000000..9d4905255 --- /dev/null +++ b/convex-db/src/main/java/convex/db/store/SegmentedEtchStore.java @@ -0,0 +1,284 @@ +package convex.db.store; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.function.Consumer; + +import convex.core.data.ACell; +import convex.core.data.Hash; +import convex.core.data.Ref; +import convex.core.store.ACachedStore; +import convex.etch.EtchStore; + +/** + * A tiered store backed by a directory of Etch segment files. + * + *

Layout inside the directory: + * 

+ *   hot.etch                    — active write target (always present)
+ *   warm-YYYYMMDD-HHmmss.etch   — sealed segment, not yet compacted
+ *   warm-YYYYMMDD-HHmmss-c.etch — sealed segment, already compacted
+ *
+ * + *

Writes always target the hot segment. Reads cascade: hot first, then sealed + * segments in reverse-timestamp order (newest first), stopping at first hit. + * + *

Call {@link #sealHot()} to rotate the hot segment into a warm segment and open + * a fresh hot file. Sealed segments are immutable after creation. + * + *

Call {@link #compactSealed(int)} or {@link #compactAllSealed()} to compact warm + * segments online (no server downtime required — warm files are never written to). + * Compacted segments are renamed with a {@code -c} suffix so they are not compacted again. + * + *

Note: when storeRef recurses into child cells, it checks only the hot segment's + * local index, so cells already present in a sealed segment may be written again to hot. + * This is harmless duplication; a compaction pass on the hot segment will deduplicate. + */ +public class SegmentedEtchStore extends ACachedStore { + + private static final String HOT_NAME = "hot.etch"; + private static final String COMPACTED_SUFFIX = "-c"; + private static final DateTimeFormatter TIMESTAMP_FMT = + DateTimeFormatter.ofPattern("yyyyMMdd-HHmmssSSS"); + + private final File directory; + /** Active write target. */ + private EtchStore hot; + /** Sealed read-only segments, index 0 = oldest, last index = newest. */ + private final List sealed; + + private SegmentedEtchStore(File dir, EtchStore hot, List sealed) { + this.directory = dir; + this.hot = hot; + this.sealed = sealed; + } + + // ── Factory methods ─────────────────────────────────────────────────────── + + /** + * Opens (or creates) a segmented store rooted at {@code dir}. + * Creates the directory and a fresh {@code hot.etch} if they do not exist. + * Any existing {@code warm-*.etch} files are opened as sealed segments. + * + * @param dir Directory for this segmented store + * @return New SegmentedEtchStore instance + * @throws IOException on IO error + */ + public static SegmentedEtchStore open(File dir) throws IOException { + dir.mkdirs(); + // Accept both warm-*.etch and warm-*-c.etch (compacted) + File[] warmFiles = dir.listFiles((d, n) -> n.startsWith("warm-") && n.endsWith(".etch")); + List sealed = new ArrayList<>(); + if (warmFiles != null) { + Arrays.sort(warmFiles, Comparator.comparing(File::getName)); + for (File f : warmFiles) sealed.add(EtchStore.create(f)); + } + EtchStore hot = EtchStore.create(new File(dir, HOT_NAME)); + return new SegmentedEtchStore(dir, hot, sealed); + } + + /** + * Creates a fresh segmented store: removes all existing {@code *.etch} files in + * {@code dir}, then opens a single empty hot segment. + * Use this at the start of each benchmark run to ensure a clean state. + * + * @param dir Directory for this segmented store + * @return New empty SegmentedEtchStore instance + * @throws IOException on IO error + */ + public static SegmentedEtchStore createFresh(File dir) throws IOException { + dir.mkdirs(); + File[] existing = dir.listFiles((d, n) -> n.endsWith(".etch")); + if (existing != null) for (File f : existing) f.delete(); + return open(dir); + } + + // ── Segment rotation ────────────────────────────────────────────────────── + + /** + * Seals the current hot segment and opens a new empty hot segment. + * + *

The current {@code hot.etch} is flushed, closed, and renamed to + * {@code warm-{timestamp}.etch}. A fresh empty {@code hot.etch} is then opened + * as the new write target. The sealed segment becomes the newest in the + * read-fallback chain. + * + * @throws IOException on IO error + */ + public synchronized void sealHot() throws IOException { + hot.flush(); + hot.close(); + String ts = LocalDateTime.now().format(TIMESTAMP_FMT); + File oldHot = new File(directory, HOT_NAME); + File warmFile = new File(directory, "warm-" + ts + ".etch"); + Files.move(oldHot.toPath(), warmFile.toPath()); + sealed.add(EtchStore.create(warmFile)); // newest = last + hot = EtchStore.create(new File(directory, HOT_NAME)); + } + + // ── Read path ───────────────────────────────────────────────────────────── + + @Override + public Ref refForHash(Hash hash) { + // 1. Shared in-memory cache (avoids redundant disk reads across all segments) + Ref cached = checkCache(hash); + if (cached != null) return cached; + + // 2. Hot segment (most likely for recent writes) + Ref r = hot.refForHash(hash); + if (r != null) return r; + + // 3. Sealed segments — newest first (most likely location for recent history) + for (int i = sealed.size() - 1; i >= 0; i--) { + r = sealed.get(i).refForHash(hash); + if (r != null) return r; + } + return null; + } + + // ── Write path ──────────────────────────────────────────────────────────── + + @Override + public Ref storeRef(Ref ref, int status, + Consumer> noveltyHandler) throws IOException { + return hot.storeRef(ref, status, noveltyHandler); + } + + @Override + public Ref storeTopRef(Ref ref, int status, + Consumer> noveltyHandler) throws IOException { + return hot.storeTopRef(ref, status, noveltyHandler); + } + + // ── Root management ─────────────────────────────────────────────────────── + + @Override + public Hash getRootHash() throws IOException { + return hot.getRootHash(); + } + + @Override + public Ref setRootData(T data) throws IOException { + return hot.setRootData(data); + } + + // ── Lifecycle ───────────────────────────────────────────────────────────── + + /** Flushes the hot segment to disk. */ + public void flush() throws IOException { + hot.flush(); + } + + @Override + public void close() { + hot.close(); + for (EtchStore s : sealed) s.close(); + } + + // ── Sealed segment compaction ───────────────────────────────────────────── + + /** + * Returns true if the sealed segment at {@code index} has already been compacted + * (its filename ends with {@code -c.etch}). + * + * @param index Index into the sealed segment list (0 = oldest) + * @return true if already compacted + */ + public boolean isCompacted(int index) { + File f = sealed.get(index).getFile(); + String name = f.getName(); + return name.endsWith(COMPACTED_SUFFIX + ".etch"); + } + + /** + * Returns true if the sealed segment at {@code index} needs compaction. + * Equivalent to {@code !isCompacted(index)}. + * + * @param index Index into the sealed segment list (0 = oldest) + * @return true if compaction has not been run on this segment + */ + public boolean needsCompaction(int index) { + return !isCompacted(index); + } + + /** + * Compacts the sealed segment at {@code index} in place. + * + *

Writes a compacted copy to a temporary file alongside the original, then + * atomically renames it to the compacted name ({@code warm-TIMESTAMP-c.etch}). + * The old segment file is deleted. The in-memory {@code sealed} list is updated + * to point at the new file. No server downtime required. + * + *

If the segment is already compacted this is a no-op. + * + * @param index Index into the sealed segment list (0 = oldest) + * @throws IOException on IO error + */ + public synchronized void compactSealed(int index) throws IOException { + if (isCompacted(index)) return; + + EtchStore src = sealed.get(index); + File srcFile = src.getFile(); + + // Derive compacted filename: strip .etch, append -c.etch + String base = srcFile.getName(); + base = base.substring(0, base.length() - ".etch".length()); + File tmpFile = new File(directory, base + "-c.tmp.etch"); + File dstFile = new File(directory, base + COMPACTED_SUFFIX + ".etch"); + + if (tmpFile.exists()) tmpFile.delete(); + + EtchStore compacted = src.compact(tmpFile); + compacted.close(); + + // Atomic swap: rename tmp → dst, delete old src, reload segment + Files.move(tmpFile.toPath(), dstFile.toPath()); + src.close(); + srcFile.delete(); + + sealed.set(index, EtchStore.create(dstFile)); + } + + /** + * Compacts all sealed segments that have not yet been compacted. + * Segments are processed oldest-first. Already-compacted segments are skipped. + * + * @throws IOException on IO error + */ + public void compactAllSealed() throws IOException { + for (int i = 0; i < sealed.size(); i++) { + if (needsCompaction(i)) compactSealed(i); + } + } + + // ── Accessors ───────────────────────────────────────────────────────────── + + /** Returns the directory backing this store. */ + public File getDirectory() { return directory; } + + /** Returns the active hot EtchStore. */ + public EtchStore getHot() { return hot; } + + /** Returns an unmodifiable view of the sealed segment stores, oldest first. */ + public List getSealed() { return Collections.unmodifiableList(sealed); } + + /** Returns the file path of the current hot segment. */ + public File getHotFile() { return new File(directory, HOT_NAME); } + + /** Returns the number of sealed segments. */ + public int getSealedCount() { return sealed.size(); } + + @Override + public String shortName() { return "Segmented:" + directory.getName(); } + + @Override + public boolean isPersistent() { return true; } +} diff --git a/convex-db/src/main/java/org/apache/calcite/jdbc/ConvexMeta.java b/convex-db/src/main/java/org/apache/calcite/jdbc/ConvexMeta.java index 1d200d662..c8113ebc8 100644 --- a/convex-db/src/main/java/org/apache/calcite/jdbc/ConvexMeta.java +++ b/convex-db/src/main/java/org/apache/calcite/jdbc/ConvexMeta.java @@ -1,6 +1,15 @@ package org.apache.calcite.jdbc; +import java.util.Collections; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + import org.apache.calcite.avatica.AvaticaConnection; +import org.apache.calcite.avatica.Meta.ExecuteResult; +import org.apache.calcite.avatica.Meta.MetaResultSet; +import org.apache.calcite.avatica.Meta.PrepareCallback; +import org.apache.calcite.avatica.Meta.StatementHandle; +import org.apache.calcite.avatica.NoSuchStatementException; import org.apache.calcite.schema.SchemaPlus; import convex.db.calcite.ConvexSchema; @@ -55,6 +64,47 @@ public static ConvexMeta create(AvaticaConnection connection) { return new ConvexMeta((CalciteConnectionImpl) connection); } + // ── Secondary index DDL interception ───────────────────────────────────── + + private static final Pattern CREATE_INDEX = Pattern.compile( + "(?i)CREATE\\s+INDEX\\s+(IF\\s+NOT\\s+EXISTS\\s+)?(\\w+)\\s+ON\\s+(\\w+)\\s*\\(\\s*(\\w+)[^)]*\\)\\s*"); + + private static final Pattern DROP_INDEX = Pattern.compile( + "(?i)DROP\\s+INDEX\\s+(IF\\s+EXISTS\\s+)?(\\w+)(?:\\s+ON\\s+(\\w+))?\\s*"); + + @Override + public ExecuteResult prepareAndExecute(StatementHandle h, String sql, + long maxRowCount, int maxRowsInFirstFrame, PrepareCallback callback) + throws NoSuchStatementException { + Matcher m = CREATE_INDEX.matcher(sql.trim()); + if (m.matches()) { + boolean ifNotExists = m.group(1) != null; + String indexName = m.group(2); + String tableName = m.group(3); + String columnName = m.group(4); + ConvexSchema schema = findConvexSchema(getSchemaName()); + if (schema != null) { + schema.createIndex(indexName, tableName, columnName, ifNotExists); + } + return new ExecuteResult(Collections.singletonList( + MetaResultSet.count(h.connectionId, h.id, 0L))); + } + + m = DROP_INDEX.matcher(sql.trim()); + if (m.matches()) { + boolean ifExists = m.group(1) != null; + String indexName = m.group(2); + ConvexSchema schema = findConvexSchema(getSchemaName()); + if (schema != null) { + schema.dropIndex(indexName, ifExists); + } + return new ExecuteResult(Collections.singletonList( + MetaResultSet.count(h.connectionId, h.id, 0L))); + } + + return super.prepareAndExecute(h, sql, maxRowCount, maxRowsInFirstFrame, callback); + } + /** Syncs fork to parent, then starts a new fork if still in manual-commit mode. */ @Override public void commit(ConnectionHandle ch) { diff --git a/convex-db/src/test/java/convex/db/SecondaryIndexMergeTest.java b/convex-db/src/test/java/convex/db/SecondaryIndexMergeTest.java new file mode 100644 index 000000000..63fdf5dad --- /dev/null +++ b/convex-db/src/test/java/convex/db/SecondaryIndexMergeTest.java @@ -0,0 +1,275 @@ +package convex.db; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Test; + +import convex.core.crypto.AKeyPair; +import convex.core.data.ABlob; +import convex.core.data.ACell; +import convex.core.data.AVector; +import convex.core.data.Index; +import convex.core.data.Strings; +import convex.core.data.Vectors; +import convex.core.data.prim.CVMLong; +import convex.db.calcite.ConvexType; +import convex.db.lattice.SQLDatabase; +import convex.db.lattice.SQLSchema; + +/** + * CRDT correctness tests for secondary column indices. + * + *

Verifies that indices remain consistent after lattice merge — the core + * property that makes secondary indices safe in a distributed setting. + * + *

Test pattern: fork two independent replicas, apply mutations on each, + * merge them, then assert that the index result equals the golden-source + * (selectAll filtered in-memory) on the merged state. + * + *

No Thread.sleep() calls are needed: the JVM-global write-sequence counter + * in SQLSchema guarantees every write gets a unique version number, so LWW ties + * are impossible within a single process. + */ +public class SecondaryIndexMergeTest { + + private static final String TBL = "employees"; + private static final String[] COLS = {"id", "status", "dept", "score"}; + private static final ConvexType[] TYPES = { + ConvexType.INTEGER, ConvexType.VARCHAR, ConvexType.VARCHAR, ConvexType.INTEGER + }; + + // ── Helpers ────────────────────────────────────────────────────────────── + + private SQLSchema freshSchema() { + SQLSchema s = SQLDatabase.create("test", AKeyPair.generate()).tables(); + s.createTable(TBL, COLS, TYPES); + return s; + } + + private void insert(SQLSchema s, long id, String status, String dept, long score) { + s.insert(TBL, Vectors.of(CVMLong.create(id), + Strings.create(status), Strings.create(dept), CVMLong.create(score))); + } + + /** + * Golden-source filter on the given schema: rows where column colIdx == value. + */ + private Index> golden(SQLSchema s, int colIdx, ACell value) { + Index> all = s.selectAll(TBL); + Index> result = Index.none(); + for (var e : all.entrySet()) { + AVector row = e.getValue(); + ACell cell = (colIdx < row.count()) ? row.get(colIdx) : null; + if (value.equals(cell)) result = result.assoc(e.getKey(), row); + } + return result; + } + + // ── Concurrent insert on disjoint keys ─────────────────────────────────── + + @Test + void mergeUnion_disjointInserts() { + SQLSchema base = freshSchema(); + base.createIndex(TBL, "status"); + + SQLSchema nodeA = base.fork(); + SQLSchema nodeB = base.fork(); + + insert(nodeA, 1, "active", "eng", 80); + insert(nodeB, 2, "active", "ops", 75); + + nodeA.sync(); + nodeB.sync(); + + ACell active = Strings.create("active"); + Index> indexed = base.selectByColumn(TBL, "status", active); + Index> golden = golden(base, 1, active); + + assertEquals(2, indexed.count(), "Merged index must contain both inserted rows"); + assertEquals(golden, indexed, "Index must match golden after merge"); + } + + // ── Concurrent insert on same key (LWW) ────────────────────────────────── + + @Test + void mergeLWW_sameKey_indexConsistentAfterMerge() { + // Global write-sequence counter ensures nodeB's write gets a strictly higher + // version than nodeA's — no sleep needed. + SQLSchema base = freshSchema(); + base.createIndex(TBL, "status"); + + SQLSchema nodeA = base.fork(); + insert(nodeA, 10, "active", "eng", 90); + SQLSchema nodeB = base.fork(); + insert(nodeB, 10, "inactive", "eng", 90); // Same PK, strictly higher counter → wins + + nodeA.sync(); + nodeB.sync(); + + ACell active = Strings.create("active"); + ACell inactive = Strings.create("inactive"); + + // nodeB's write must win (later counter) + AVector row = base.selectByKey(TBL, CVMLong.create(10)); + assertNotNull(row); + assertEquals(Strings.create("inactive"), row.get(1), + "nodeB's 'inactive' must win LWW (later counter)"); + + // Index must be consistent with golden source + assertEquals(golden(base, 1, active), base.selectByColumn(TBL, "status", active)); + assertEquals(golden(base, 1, inactive), base.selectByColumn(TBL, "status", inactive)); + + // Exactly one row for pk=10 + long total = base.selectByColumn(TBL, "status", active).count() + + base.selectByColumn(TBL, "status", inactive).count(); + assertEquals(1, total, "Exactly one value must survive LWW merge"); + } + + // ── Tombstone propagation ───────────────────────────────────────────────── + + @Test + void tombstone_propagatesAcrossMerge() { + SQLSchema base = freshSchema(); + base.createIndex(TBL, "status"); + + insert(base, 20, "active", "eng", 70); + insert(base, 21, "active", "ops", 80); + + SQLSchema nodeA = base.fork(); + SQLSchema nodeB = base.fork(); + + nodeA.deleteByKey(TBL, CVMLong.create(20)); + insert(nodeB, 22, "active", "hr", 85); + + nodeA.sync(); + nodeB.sync(); + + assertNull(base.selectByKey(TBL, CVMLong.create(20)), "Row 20 must be deleted"); + assertNotNull(base.selectByKey(TBL, CVMLong.create(21))); + assertNotNull(base.selectByKey(TBL, CVMLong.create(22))); + + ACell active = Strings.create("active"); + assertEquals(2, base.selectByColumn(TBL, "status", active).count(), + "Index must exclude deleted row 20"); + assertEquals(golden(base, 1, active), base.selectByColumn(TBL, "status", active), + "Index must match golden after tombstone propagation"); + } + + // ── Tombstone wins on equal version (via mergeReplicas) ────────────────── + + @Test + void tombstone_winsOnEqualTimestamp() { + AKeyPair kpA = AKeyPair.generate(); + AKeyPair kpB = AKeyPair.generate(); + SQLDatabase dbA = SQLDatabase.create("shared", kpA); + SQLDatabase dbB = SQLDatabase.create("shared", kpB); + + dbA.tables().createTable(TBL, COLS, TYPES); + dbB.tables().createTable(TBL, COLS, TYPES); + dbA.tables().createIndex(TBL, "status"); + dbB.tables().createIndex(TBL, "status"); + + insert(dbA.tables(), 30, "active", "eng", 60); + + // Propagate row 30 to dbB + dbB.mergeReplicas(dbA.exportReplica()); + assertNotNull(dbB.tables().selectByKey(TBL, CVMLong.create(30))); + + // dbB deletes row 30 + dbB.tables().deleteByKey(TBL, CVMLong.create(30)); + + // Merge dbB back into dbA — tombstone must propagate + dbA.mergeReplicas(dbB.exportReplica()); + + assertNull(dbA.tables().selectByKey(TBL, CVMLong.create(30))); + + ACell active = Strings.create("active"); + assertEquals(golden(dbA.tables(), 1, active), + dbA.tables().selectByColumn(TBL, "status", active), + "Index must reflect tombstone after merge"); + } + + // ── Merge idempotency ───────────────────────────────────────────────────── + + @Test + void mergeIdempotent() { + SQLSchema base = freshSchema(); + base.createIndex(TBL, "dept"); + + insert(base, 40, "active", "eng", 70); + insert(base, 41, "active", "ops", 80); + + SQLSchema fork = base.fork(); + insert(fork, 42, "inactive", "eng", 65); + fork.sync(); + fork.sync(); // double-sync must be a no-op + + ACell eng = Strings.create("eng"); + assertEquals(golden(base, 2, eng), base.selectByColumn(TBL, "dept", eng), + "Double-merge must be idempotent"); + } + + // ── Index consistency on multiple columns ───────────────────────────────── + + @Test + void multipleIndices_independentlyConsistent() { + SQLSchema base = freshSchema(); + base.createIndex(TBL, "status"); + base.createIndex(TBL, "dept"); + + SQLSchema nodeA = base.fork(); + SQLSchema nodeB = base.fork(); + + insert(nodeA, 50, "active", "eng", 90); + insert(nodeA, 51, "inactive", "ops", 60); + insert(nodeB, 52, "active", "eng", 75); + insert(nodeB, 53, "active", "hr", 85); + + nodeA.sync(); + nodeB.sync(); + + ACell active = Strings.create("active"); + ACell eng = Strings.create("eng"); + + assertEquals(golden(base, 1, active), base.selectByColumn(TBL, "status", active), + "status index must be consistent after merge"); + assertEquals(golden(base, 2, eng), base.selectByColumn(TBL, "dept", eng), + "dept index must be consistent after merge"); + } + + // ── Fuzz: random inserts and deletes ───────────────────────────────────── + + @Test + void fuzz_randomInsertsAndDeletes_indexConsistent() { + SQLSchema base = freshSchema(); + base.createIndex(TBL, "status"); + + java.util.Random rnd = new java.util.Random(42); + String[] statuses = {"active", "inactive", "pending"}; + + SQLSchema nodeA = base.fork(); + SQLSchema nodeB = base.fork(); + + for (int i = 0; i < 30; i++) { + insert(nodeA, 1000 + i, statuses[rnd.nextInt(statuses.length)], "eng", 50 + rnd.nextInt(50)); + } + for (int i = 0; i < 10; i++) { + nodeA.deleteByKey(TBL, CVMLong.create(1000 + rnd.nextInt(30))); + } + for (int i = 0; i < 30; i++) { + insert(nodeB, 2000 + i, statuses[rnd.nextInt(statuses.length)], "ops", 50 + rnd.nextInt(50)); + } + for (int i = 0; i < 10; i++) { + nodeB.deleteByKey(TBL, CVMLong.create(2000 + rnd.nextInt(30))); + } + + nodeA.sync(); + nodeB.sync(); + + for (String st : statuses) { + ACell val = Strings.create(st); + assertEquals(golden(base, 1, val), base.selectByColumn(TBL, "status", val), + "Fuzz: index inconsistent for status=" + st); + } + } +} diff --git a/convex-db/src/test/java/convex/db/SecondaryIndexTest.java b/convex-db/src/test/java/convex/db/SecondaryIndexTest.java new file mode 100644 index 000000000..c196b3657 --- /dev/null +++ b/convex-db/src/test/java/convex/db/SecondaryIndexTest.java @@ -0,0 +1,318 @@ +package convex.db; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import convex.core.crypto.AKeyPair; +import convex.core.data.ABlob; +import convex.core.data.ACell; +import convex.core.data.AVector; +import convex.core.data.Index; +import convex.core.data.Strings; +import convex.core.data.Vectors; +import convex.core.data.prim.CVMLong; +import convex.db.calcite.ConvexType; +import convex.db.lattice.SQLDatabase; +import convex.db.lattice.SQLSchema; + +/** + * Lattice-level unit tests for secondary column indices. + * + *

Tests the index lifecycle (create/drop) and correctness of + * exact-match and range lookups via the index, always verified against + * the golden-source: selectAll() filtered in-memory. + * + *

These tests define the API contract and will be RED until the + * secondary index feature is implemented. + */ +public class SecondaryIndexTest { + + private SQLSchema tables; + + /** employees(id INTEGER, status VARCHAR, dept VARCHAR, score INTEGER) */ + private static final String TBL = "employees"; + private static final String[] COLS = {"id", "status", "dept", "score"}; + private static final ConvexType[] TYPES = { + ConvexType.INTEGER, ConvexType.VARCHAR, ConvexType.VARCHAR, ConvexType.INTEGER + }; + + @BeforeEach + void setUp() { + tables = SQLDatabase.create("test", AKeyPair.generate()).tables(); + tables.createTable(TBL, COLS, TYPES); + } + + // ── Helpers ────────────────────────────────────────────────────────────── + + private void insertEmployee(long id, String status, String dept, long score) { + tables.insert(TBL, Vectors.of(CVMLong.create(id), + Strings.create(status), Strings.create(dept), CVMLong.create(score))); + } + + /** + * Golden-source filter: returns all live rows where column {@code colIdx} + * equals {@code value}, keyed by primary-key blob. + */ + private Index> filterByColumn(int colIdx, ACell value) { + Index> all = tables.selectAll(TBL); + Index> result = Index.none(); + for (var e : all.entrySet()) { + AVector row = e.getValue(); + ACell cell = (colIdx < row.count()) ? row.get(colIdx) : null; + if (value.equals(cell)) { + result = result.assoc(e.getKey(), row); + } + } + return result; + } + + // ── Index lifecycle ────────────────────────────────────────────────────── + + @Test + void createIndex_returnsTrue() { + assertTrue(tables.createIndex(TBL, "status"), + "createIndex should succeed on existing table"); + } + + @Test + void createIndex_idempotent() { + assertTrue(tables.createIndex(TBL, "status")); + // Second call on the same column should return false (already exists) + assertFalse(tables.createIndex(TBL, "status"), + "createIndex on an already-indexed column should return false"); + } + + @Test + void createIndex_nonExistentTable_returnsFalse() { + assertFalse(tables.createIndex("ghost", "status")); + } + + @Test + void createIndex_nonExistentColumn_returnsFalse() { + assertFalse(tables.createIndex(TBL, "no_such_col")); + } + + @Test + void hasIndex_falseBeforeCreate() { + assertFalse(tables.hasIndex(TBL, "status")); + } + + @Test + void hasIndex_trueAfterCreate() { + tables.createIndex(TBL, "status"); + assertTrue(tables.hasIndex(TBL, "status")); + } + + @Test + void dropIndex_trueAfterCreate() { + tables.createIndex(TBL, "status"); + assertTrue(tables.dropIndex(TBL, "status")); + assertFalse(tables.hasIndex(TBL, "status")); + } + + @Test + void dropIndex_falseIfNotPresent() { + assertFalse(tables.dropIndex(TBL, "status")); + } + + // ── Index populated on existing data ───────────────────────────────────── + + @Test + void createIndex_populatesFromExistingRows() { + insertEmployee(1, "active", "eng", 90); + insertEmployee(2, "inactive", "eng", 70); + insertEmployee(3, "active", "hr", 80); + + tables.createIndex(TBL, "status"); + + ACell active = Strings.create("active"); + Index> indexed = tables.selectByColumn(TBL, "status", active); + Index> golden = filterByColumn(1, active); + + assertEquals(golden.count(), indexed.count(), + "Index lookup count must match golden filter count"); + assertEquals(golden, indexed, + "Index lookup must match golden filter result"); + } + + // ── Insert updates index ───────────────────────────────────────────────── + + @Test + void insertUpdatesIndex() { + tables.createIndex(TBL, "status"); + insertEmployee(10, "active", "eng", 85); + insertEmployee(11, "active", "ops", 75); + insertEmployee(12, "inactive", "eng", 60); + + ACell active = Strings.create("active"); + Index> indexed = tables.selectByColumn(TBL, "status", active); + Index> golden = filterByColumn(1, active); + + assertEquals(golden.count(), indexed.count()); + assertEquals(golden, indexed); + } + + @Test + void insertAllUpdatesIndex() { + tables.createIndex(TBL, "dept"); + + List> rows = new ArrayList<>(); + for (int i = 0; i < 20; i++) { + String dept = (i % 3 == 0) ? "eng" : (i % 3 == 1) ? "ops" : "hr"; + rows.add(Vectors.of(CVMLong.create(i), Strings.create("active"), + Strings.create(dept), CVMLong.create(50 + i))); + } + tables.insertAll(TBL, rows); + + ACell eng = Strings.create("eng"); + Index> indexed = tables.selectByColumn(TBL, "dept", eng); + Index> golden = filterByColumn(2, eng); + + assertEquals(golden.count(), indexed.count()); + assertEquals(golden, indexed); + } + + // ── Delete updates index ───────────────────────────────────────────────── + + @Test + void deleteUpdatesIndex() { + tables.createIndex(TBL, "status"); + insertEmployee(20, "active", "eng", 88); + insertEmployee(21, "active", "ops", 72); + + // Delete row 20 + tables.deleteByKey(TBL, CVMLong.create(20)); + + ACell active = Strings.create("active"); + Index> indexed = tables.selectByColumn(TBL, "status", active); + Index> golden = filterByColumn(1, active); + + assertEquals(golden.count(), indexed.count()); + assertEquals(golden, indexed); + } + + @Test + void deleteNonExistentDoesNotCorruptIndex() { + tables.createIndex(TBL, "status"); + insertEmployee(30, "active", "eng", 99); + + // Deleting a non-existent key must not corrupt the index + tables.deleteByKey(TBL, CVMLong.create(9999)); + + ACell active = Strings.create("active"); + assertEquals(filterByColumn(1, active), tables.selectByColumn(TBL, "status", active)); + } + + // ── Exact lookup ───────────────────────────────────────────────────────── + + @Test + void lookupExact_noMatch() { + tables.createIndex(TBL, "status"); + insertEmployee(40, "active", "eng", 70); + + Index> result = + tables.selectByColumn(TBL, "status", Strings.create("pending")); + assertTrue(result.count() == 0, "Lookup for absent value should return empty"); + } + + @Test + void lookupExact_multipleMatches() { + tables.createIndex(TBL, "dept"); + insertEmployee(50, "active", "eng", 80); + insertEmployee(51, "inactive", "eng", 65); + insertEmployee(52, "active", "hr", 90); + + ACell eng = Strings.create("eng"); + Index> indexed = tables.selectByColumn(TBL, "dept", eng); + Index> golden = filterByColumn(2, eng); + + assertEquals(2, indexed.count()); + assertEquals(golden, indexed); + } + + @Test + void lookupExact_integerColumn() { + tables.createIndex(TBL, "score"); + insertEmployee(60, "active", "eng", 100); + insertEmployee(61, "active", "ops", 100); + insertEmployee(62, "active", "hr", 90); + + ACell score100 = CVMLong.create(100); + Index> indexed = tables.selectByColumn(TBL, "score", score100); + Index> golden = filterByColumn(3, score100); + + assertEquals(2, indexed.count()); + assertEquals(golden, indexed); + } + + // ── Range lookup ───────────────────────────────────────────────────────── + + @Test + void lookupRange_integer() { + tables.createIndex(TBL, "score"); + for (int i = 0; i < 10; i++) { + insertEmployee(100 + i, "active", "eng", 60 + i * 5); // 60,65,70,..105 + } + + // Range [70, 90] inclusive + Index> indexed = + tables.selectByColumnRange(TBL, "score", CVMLong.create(70), CVMLong.create(90)); + + // Golden: filter all rows with 70 ≤ score ≤ 90 + Index> all = tables.selectAll(TBL); + Index> golden = Index.none(); + for (var e : all.entrySet()) { + AVector row = e.getValue(); + long s = ((CVMLong) row.get(3)).longValue(); + if (s >= 70 && s <= 90) golden = golden.assoc(e.getKey(), row); + } + + assertEquals(golden.count(), indexed.count()); + assertEquals(golden, indexed); + } + + @Test + void lookupRange_emptyResult() { + tables.createIndex(TBL, "score"); + insertEmployee(200, "active", "eng", 50); + + // Range [80, 100] — no rows in that range + Index> result = + tables.selectByColumnRange(TBL, "score", CVMLong.create(80), CVMLong.create(100)); + assertTrue(result.count() == 0); + } + + // ── No index: fallback behaviour ───────────────────────────────────────── + + @Test + void lookupWithoutIndex_stillWorksViaScan() { + // selectByColumn should work even without an explicit index (full scan fallback) + insertEmployee(300, "active", "eng", 70); + insertEmployee(301, "inactive", "eng", 80); + + ACell active = Strings.create("active"); + Index> result = tables.selectByColumn(TBL, "status", active); + Index> golden = filterByColumn(1, active); + + assertEquals(golden.count(), result.count()); + assertEquals(golden, result); + } + + // ── selectByColumn on dropped table ────────────────────────────────────── + + @Test + void lookupOnDroppedTable_returnsEmpty() { + tables.createIndex(TBL, "status"); + insertEmployee(400, "active", "eng", 70); + tables.dropTable(TBL); + + Index> result = + tables.selectByColumn(TBL, "status", Strings.create("active")); + assertEquals(0, result.count()); + } +} diff --git a/convex-db/src/test/java/convex/db/VersionedSQLSchemaTest.java b/convex-db/src/test/java/convex/db/VersionedSQLSchemaTest.java new file mode 100644 index 000000000..2f1e66612 --- /dev/null +++ b/convex-db/src/test/java/convex/db/VersionedSQLSchemaTest.java @@ -0,0 +1,297 @@ +package convex.db; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import convex.core.data.ACell; +import convex.core.data.AVector; +import convex.core.data.Vectors; +import convex.core.data.prim.CVMLong; +import convex.db.lattice.VersionedSQLSchema; +import convex.db.lattice.VersionedSQLTable; + +/** + * Unit tests for {@link VersionedSQLSchema} and {@link VersionedSQLTable}. + * + *

Covers: insert, update, deduplication, delete, history ordering, + * point-in-time (AS OF) queries, live row count, and history merge. + */ +public class VersionedSQLSchemaTest { + + private static final String TBL = "items"; + + private VersionedSQLSchema schema; + + @BeforeEach + void setUp() { + schema = VersionedSQLSchema.create(); + schema.createTable(TBL, new String[]{"id", "name", "value"}); + } + + // ── Helpers ─────────────────────────────────────────────────────────────── + + AVector row(long id, String name, long value) { + return Vectors.of(CVMLong.create(id), convex.core.data.Strings.create(name), CVMLong.create(value)); + } + + CVMLong pk(long id) { return CVMLong.create(id); } + + // ── Insert ──────────────────────────────────────────────────────────────── + + @Test + void testInsertCreatesLiveRow() { + schema.insert(TBL, row(1, "alpha", 10)); + AVector found = schema.selectByKey(TBL, pk(1)); + assertNotNull(found); + } + + @Test + void testInsertRecordsHistoryEntry() { + schema.insert(TBL, row(1, "alpha", 10)); + List> history = schema.getHistory(TBL, pk(1)); + assertEquals(1, history.size()); + assertEquals(VersionedSQLTable.CT_INSERT, changeType(history.get(0))); + } + + @Test + void testInsertRowCount() { + schema.insert(TBL, row(1, "alpha", 10)); + schema.insert(TBL, row(2, "beta", 20)); + assertEquals(2, schema.getRowCount(TBL)); + } + + // ── Update ──────────────────────────────────────────────────────────────── + + @Test + void testUpdateChangesLiveRow() { + schema.insert(TBL, row(1, "alpha", 10)); + schema.insert(TBL, row(1, "alpha", 99)); + + AVector found = schema.selectByKey(TBL, pk(1)); + assertNotNull(found); + assertEquals(CVMLong.create(99), VersionedSQLTable.getHistoryValues( + schema.getHistory(TBL, pk(1)).get(1)).get(2)); + } + + @Test + void testUpdateRecordsUpdateEntry() { + schema.insert(TBL, row(1, "alpha", 10)); + schema.insert(TBL, row(1, "alpha", 99)); + + List> history = schema.getHistory(TBL, pk(1)); + assertEquals(2, history.size()); + assertEquals(VersionedSQLTable.CT_INSERT, changeType(history.get(0))); + assertEquals(VersionedSQLTable.CT_UPDATE, changeType(history.get(1))); + } + + @Test + void testUpdateDoesNotChangeRowCount() { + schema.insert(TBL, row(1, "alpha", 10)); + schema.insert(TBL, row(1, "alpha", 99)); + assertEquals(1, schema.getRowCount(TBL)); + } + + // ── Deduplication ───────────────────────────────────────────────────────── + + @Test + void testDuplicateInsertSkipped() { + schema.insert(TBL, row(1, "alpha", 10)); + boolean written = schema.insert(TBL, row(1, "alpha", 10)); + assertFalse(written, "Identical re-insert should be skipped"); + } + + @Test + void testDuplicateInsertNoExtraHistory() { + schema.insert(TBL, row(1, "alpha", 10)); + schema.insert(TBL, row(1, "alpha", 10)); // duplicate + assertEquals(1, schema.getHistory(TBL, pk(1)).size()); + } + + // ── Delete ──────────────────────────────────────────────────────────────── + + @Test + void testDeleteRemovesLiveRow() { + schema.insert(TBL, row(1, "alpha", 10)); + schema.deleteByKey(TBL, pk(1)); + assertNull(schema.selectByKey(TBL, pk(1))); + } + + @Test + void testDeleteDecrementsRowCount() { + schema.insert(TBL, row(1, "alpha", 10)); + schema.insert(TBL, row(2, "beta", 20)); + schema.deleteByKey(TBL, pk(1)); + assertEquals(1, schema.getRowCount(TBL)); + } + + @Test + void testDeleteRecordsDeleteEntry() { + schema.insert(TBL, row(1, "alpha", 10)); + schema.deleteByKey(TBL, pk(1)); + + List> history = schema.getHistory(TBL, pk(1)); + assertEquals(2, history.size()); + assertEquals(VersionedSQLTable.CT_DELETE, changeType(history.get(1))); + } + + @Test + void testDeleteNonExistentIsNoop() { + boolean deleted = schema.deleteByKey(TBL, pk(99)); + assertFalse(deleted); + assertTrue(schema.getHistory(TBL, pk(99)).isEmpty()); + } + + @Test + void testDeleteThenInsertAddsNewHistory() { + schema.insert(TBL, row(1, "alpha", 10)); + schema.deleteByKey(TBL, pk(1)); + schema.insert(TBL, row(1, "alpha", 20)); // re-insert + + List> history = schema.getHistory(TBL, pk(1)); + assertEquals(3, history.size()); + assertEquals(VersionedSQLTable.CT_INSERT, changeType(history.get(0))); + assertEquals(VersionedSQLTable.CT_DELETE, changeType(history.get(1))); + assertEquals(VersionedSQLTable.CT_INSERT, changeType(history.get(2))); + assertNotNull(schema.selectByKey(TBL, pk(1))); + } + + // ── History ordering ────────────────────────────────────────────────────── + + @Test + void testHistoryOldestFirst() { + schema.insert(TBL, row(1, "alpha", 10)); + schema.insert(TBL, row(1, "alpha", 20)); + schema.insert(TBL, row(1, "alpha", 30)); + + List> history = schema.getHistory(TBL, pk(1)); + assertEquals(3, history.size()); + // Nanotimes should be monotonically increasing + long prev = Long.MIN_VALUE; + for (AVector entry : history) { + long ts = ((CVMLong) entry.get(1)).longValue(); + assertTrue(ts >= prev, "History entries should be ordered oldest-first"); + prev = ts; + } + } + + @Test + void testHistoryIsolatedByPk() { + schema.insert(TBL, row(1, "alpha", 10)); + schema.insert(TBL, row(1, "alpha", 20)); + schema.insert(TBL, row(2, "beta", 99)); + + assertEquals(2, schema.getHistory(TBL, pk(1)).size()); + assertEquals(1, schema.getHistory(TBL, pk(2)).size()); + } + + @Test + void testHistoryEmptyForUnknownPk() { + assertTrue(schema.getHistory(TBL, pk(999)).isEmpty()); + } + + // ── AS OF (point-in-time) ───────────────────────────────────────────────── + + @Test + void testGetAsOfReturnsVersionAtTimestamp() { + long t1 = System.nanoTime(); + schema.insert(TBL, row(1, "alpha", 10)); + long t2 = System.nanoTime(); + schema.insert(TBL, row(1, "alpha", 20)); + long t3 = System.nanoTime(); + + // AS OF before first insert: nothing + assertNull(schema.getAsOf(TBL, pk(1), t1 - 1)); + + // AS OF between first and second insert: first version + AVector v1 = schema.getAsOf(TBL, pk(1), t2 - 1); + assertNotNull(v1); + AVector vals1 = VersionedSQLTable.getHistoryValues(v1); + assertEquals(CVMLong.create(10), vals1.get(2)); + + // AS OF after second insert: second version + AVector v2 = schema.getAsOf(TBL, pk(1), t3); + assertNotNull(v2); + AVector vals2 = VersionedSQLTable.getHistoryValues(v2); + assertEquals(CVMLong.create(20), vals2.get(2)); + } + + @Test + void testGetAsOfAfterDeleteShowsDeleteEntry() { + schema.insert(TBL, row(1, "alpha", 10)); + long tDelete = System.nanoTime(); + schema.deleteByKey(TBL, pk(1)); + long tAfter = System.nanoTime(); + + AVector entry = schema.getAsOf(TBL, pk(1), tAfter); + assertNotNull(entry); + assertEquals(VersionedSQLTable.CT_DELETE, changeType(entry)); + assertNull(VersionedSQLTable.getHistoryValues(entry)); + } + + @Test + void testGetAsOfNullForUnknownPk() { + assertNull(schema.getAsOf(TBL, pk(999), Long.MAX_VALUE)); + } + + // ── History values ──────────────────────────────────────────────────────── + + @Test + void testGetHistoryValuesMatchesInsertedRow() { + AVector inserted = row(1, "alpha", 42); + schema.insert(TBL, inserted); + + List> history = schema.getHistory(TBL, pk(1)); + AVector vals = VersionedSQLTable.getHistoryValues(history.get(0)); + assertNotNull(vals); + assertEquals(inserted, vals); + } + + @Test + void testGetHistoryValuesNullForDelete() { + schema.insert(TBL, row(1, "alpha", 10)); + schema.deleteByKey(TBL, pk(1)); + + List> history = schema.getHistory(TBL, pk(1)); + AVector deleteEntry = history.get(1); + assertNull(VersionedSQLTable.getHistoryValues(deleteEntry)); + } + + // ── Merge (CRDT union) ──────────────────────────────────────────────────── + + @Test + void testMergeUnionHistory() { + // Replica A: insert row 1 + VersionedSQLSchema replicaA = VersionedSQLSchema.create(); + replicaA.createTable(TBL, new String[]{"id", "name", "value"}); + replicaA.insert(TBL, row(1, "alpha", 10)); + + // Replica B: insert row 2 + VersionedSQLSchema replicaB = VersionedSQLSchema.create(); + replicaB.createTable(TBL, new String[]{"id", "name", "value"}); + replicaB.insert(TBL, row(2, "beta", 20)); + + // Merge B into A + replicaA.cursor().merge(replicaB.cursor().get()); + + // Both rows and both histories should be present + assertNotNull(replicaA.selectByKey(TBL, pk(1))); + assertNotNull(replicaA.selectByKey(TBL, pk(2))); + assertEquals(1, replicaA.getHistory(TBL, pk(1)).size()); + assertEquals(1, replicaA.getHistory(TBL, pk(2)).size()); + assertEquals(2, replicaA.getRowCount(TBL)); + } + + // ── Private helpers ─────────────────────────────────────────────────────── + + static long changeType(AVector histEntry) { + return ((CVMLong) histEntry.get(2)).longValue(); + } +} diff --git a/convex-db/src/test/java/convex/db/calcite/SecondaryIndexSQLTest.java b/convex-db/src/test/java/convex/db/calcite/SecondaryIndexSQLTest.java new file mode 100644 index 000000000..a6ad31636 --- /dev/null +++ b/convex-db/src/test/java/convex/db/calcite/SecondaryIndexSQLTest.java @@ -0,0 +1,291 @@ +package convex.db.calcite; + +import static org.junit.jupiter.api.Assertions.*; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import convex.db.ConvexDB; +import convex.db.lattice.SQLDatabase; + +/** + * SQL-level integration tests for secondary column indices. + * + *

Tests the full stack: DDL (CREATE INDEX / DROP INDEX), DML (INSERT/DELETE), + * and SELECT with WHERE predicates. All results are verified against the + * golden-source: the same SELECT without the index (full-table-scan path). + * + *

These tests will be RED until the secondary index feature is implemented. + */ +public class SecondaryIndexSQLTest { + + private static int counter = 0; + + private ConvexDB cdb; + private SQLDatabase db; + private Connection conn; + private String dbName; + + @BeforeEach + void setUp() throws Exception { + dbName = "idx_test_" + (++counter) + "_" + System.currentTimeMillis(); + cdb = ConvexDB.create(); + db = cdb.database(dbName); + cdb.register(dbName); + + db.tables().createTable("employees", + new String[]{"id", "status", "dept", "score"}, + new ConvexType[]{ConvexType.INTEGER, ConvexType.VARCHAR, ConvexType.VARCHAR, ConvexType.INTEGER}); + + conn = DriverManager.getConnection("jdbc:convex:database=" + dbName); + } + + @AfterEach + void tearDown() throws Exception { + if (conn != null) conn.close(); + if (cdb != null) cdb.unregister(dbName); + } + + // ── Helpers ────────────────────────────────────────────────────────────── + + private void insertRow(Statement s, int id, String status, String dept, int score) + throws SQLException { + s.executeUpdate(String.format( + "INSERT INTO employees VALUES (%d, '%s', '%s', %d)", id, status, dept, score)); + } + + /** + * Collects (id, status, dept, score) rows from a ResultSet. + */ + private List collectRows(ResultSet rs) throws SQLException { + List rows = new ArrayList<>(); + while (rs.next()) { + rows.add(rs.getInt("id") + "|" + rs.getString("status") + + "|" + rs.getString("dept") + "|" + rs.getInt("score")); + } + rows.sort(String::compareTo); + return rows; + } + + // ── DDL ────────────────────────────────────────────────────────────────── + + @Test + void createIndex_ddl() throws SQLException { + try (Statement s = conn.createStatement()) { + // Should execute without error + s.execute("CREATE INDEX idx_status ON employees (status)"); + } + assertTrue(db.tables().hasIndex("employees", "status"), + "Lattice-level index must be created by DDL"); + } + + @Test + void createIndex_ifNotExists_noError() throws SQLException { + try (Statement s = conn.createStatement()) { + s.execute("CREATE INDEX idx_status ON employees (status)"); + // IF NOT EXISTS variant should not throw + s.execute("CREATE INDEX IF NOT EXISTS idx_status ON employees (status)"); + } + } + + @Test + void dropIndex_ddl() throws SQLException { + try (Statement s = conn.createStatement()) { + s.execute("CREATE INDEX idx_dept ON employees (dept)"); + s.execute("DROP INDEX idx_dept"); + } + assertFalse(db.tables().hasIndex("employees", "dept"), + "Lattice-level index must be removed by DROP INDEX DDL"); + } + + // ── Exact-match WHERE with index ───────────────────────────────────────── + + @Test + void whereExact_matchesFullScanResult() throws SQLException { + try (Statement s = conn.createStatement()) { + s.execute("CREATE INDEX idx_status ON employees (status)"); + insertRow(s, 1, "active", "eng", 90); + insertRow(s, 2, "inactive", "eng", 70); + insertRow(s, 3, "active", "ops", 80); + insertRow(s, 4, "pending", "hr", 60); + + // Index-backed query + List indexed = collectRows( + s.executeQuery("SELECT id, status, dept, score FROM employees WHERE status = 'active'")); + + // Golden: drop the index and do a full scan + s.execute("DROP INDEX idx_status"); + List golden = collectRows( + s.executeQuery("SELECT id, status, dept, score FROM employees WHERE status = 'active'")); + + assertEquals(golden, indexed, "WHERE exact must match full-scan golden result"); + } + } + + @Test + void whereExact_noMatch_emptyResult() throws SQLException { + try (Statement s = conn.createStatement()) { + s.execute("CREATE INDEX idx_status ON employees (status)"); + insertRow(s, 10, "active", "eng", 70); + insertRow(s, 11, "active", "ops", 80); + + ResultSet rs = s.executeQuery( + "SELECT id FROM employees WHERE status = 'pending'"); + assertFalse(rs.next(), "Query for absent value must return empty result set"); + } + } + + @Test + void whereExact_integerColumn() throws SQLException { + try (Statement s = conn.createStatement()) { + s.execute("CREATE INDEX idx_score ON employees (score)"); + insertRow(s, 20, "active", "eng", 100); + insertRow(s, 21, "active", "ops", 100); + insertRow(s, 22, "active", "hr", 90); + + List indexed = collectRows( + s.executeQuery("SELECT id, status, dept, score FROM employees WHERE score = 100")); + + s.execute("DROP INDEX idx_score"); + List golden = collectRows( + s.executeQuery("SELECT id, status, dept, score FROM employees WHERE score = 100")); + + assertEquals(2, indexed.size()); + assertEquals(golden, indexed); + } + } + + // ── Range WHERE with index ──────────────────────────────────────────────── + + @Test + void whereRange_integer() throws SQLException { + try (Statement s = conn.createStatement()) { + s.execute("CREATE INDEX idx_score ON employees (score)"); + for (int i = 0; i < 10; i++) { + insertRow(s, 30 + i, "active", "eng", 60 + i * 5); // 60,65,70,...105 + } + + List indexed = collectRows(s.executeQuery( + "SELECT id, status, dept, score FROM employees WHERE score >= 70 AND score <= 90")); + + s.execute("DROP INDEX idx_score"); + List golden = collectRows(s.executeQuery( + "SELECT id, status, dept, score FROM employees WHERE score >= 70 AND score <= 90")); + + assertFalse(indexed.isEmpty(), "Range query must return results"); + assertEquals(golden, indexed, "Range result must match full-scan golden"); + } + } + + // ── DML updates index ───────────────────────────────────────────────────── + + @Test + void delete_reflectedInIndex() throws SQLException { + try (Statement s = conn.createStatement()) { + s.execute("CREATE INDEX idx_dept ON employees (dept)"); + insertRow(s, 40, "active", "eng", 85); + insertRow(s, 41, "active", "eng", 75); + insertRow(s, 42, "active", "ops", 70); + + s.executeUpdate("DELETE FROM employees WHERE id = 40"); + + List indexed = collectRows( + s.executeQuery("SELECT id, status, dept, score FROM employees WHERE dept = 'eng'")); + + s.execute("DROP INDEX idx_dept"); + List golden = collectRows( + s.executeQuery("SELECT id, status, dept, score FROM employees WHERE dept = 'eng'")); + + assertEquals(golden, indexed, "Index must reflect deletion"); + } + } + + // ── COUNT aggregate with index ──────────────────────────────────────────── + + @Test + void countAggregate_withIndex() throws SQLException { + try (Statement s = conn.createStatement()) { + s.execute("CREATE INDEX idx_status ON employees (status)"); + insertRow(s, 50, "active", "eng", 90); + insertRow(s, 51, "active", "ops", 80); + insertRow(s, 52, "inactive", "hr", 70); + + ResultSet rs = s.executeQuery( + "SELECT COUNT(*) FROM employees WHERE status = 'active'"); + assertTrue(rs.next()); + assertEquals(2, rs.getInt(1)); + } + } + + // ── Multiple indices on same table ──────────────────────────────────────── + + @Test + void multipleIndices_onSameTable() throws SQLException { + try (Statement s = conn.createStatement()) { + s.execute("CREATE INDEX idx_status ON employees (status)"); + s.execute("CREATE INDEX idx_dept ON employees (dept)"); + + insertRow(s, 60, "active", "eng", 80); + insertRow(s, 61, "inactive", "eng", 65); + insertRow(s, 62, "active", "ops", 75); + + // Query on status index + List byStatus = collectRows( + s.executeQuery("SELECT id, status, dept, score FROM employees WHERE status = 'active'")); + // Query on dept index + List byDept = collectRows( + s.executeQuery("SELECT id, status, dept, score FROM employees WHERE dept = 'eng'")); + + // Verify against scan (drop both indices) + s.execute("DROP INDEX idx_status"); + s.execute("DROP INDEX idx_dept"); + + List goldenStatus = collectRows( + s.executeQuery("SELECT id, status, dept, score FROM employees WHERE status = 'active'")); + List goldenDept = collectRows( + s.executeQuery("SELECT id, status, dept, score FROM employees WHERE dept = 'eng'")); + + assertEquals(goldenStatus, byStatus); + assertEquals(goldenDept, byDept); + } + } + + // ── Planner uses index: no full-table scan ──────────────────────────────── + + @Test + void plannerUsesIndex_notFullScan() throws SQLException { + // Insert a large number of rows so that a full scan would touch many blocks. + // After creating an index and querying for a rare value, the number of rows + // returned must equal the expected count — but more importantly, this test + // documents the expectation that the planner chooses the index path. + // (Verifying the actual plan requires EXPLAIN support — added here as a + // placeholder; the assertion falls back to result correctness.) + try (Statement s = conn.createStatement()) { + for (int i = 0; i < 500; i++) { + String st = (i == 250) ? "rare" : "common"; + insertRow(s, i, st, "eng", i); + } + s.execute("CREATE INDEX idx_status ON employees (status)"); + + List result = collectRows( + s.executeQuery("SELECT id, status, dept, score FROM employees WHERE status = 'rare'")); + + // Drop index to get golden + s.execute("DROP INDEX idx_status"); + List golden = collectRows( + s.executeQuery("SELECT id, status, dept, score FROM employees WHERE status = 'rare'")); + + assertEquals(1, result.size()); + assertEquals(golden, result); + } + } +} diff --git a/convex-db/src/test/java/convex/db/lattice/HistoryKeyTest.java b/convex-db/src/test/java/convex/db/lattice/HistoryKeyTest.java new file mode 100644 index 000000000..43c86079c --- /dev/null +++ b/convex-db/src/test/java/convex/db/lattice/HistoryKeyTest.java @@ -0,0 +1,141 @@ +package convex.db.lattice; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import convex.core.data.ABlob; +import convex.core.data.Blob; +import convex.core.data.prim.CVMLong; + +/** + * Unit tests for HistoryKey encoding, prefix matching and nanotime extraction. + */ +public class HistoryKeyTest { + + /** Encodes a CVMLong pk to blob the same way SQLSchema does. */ + static ABlob pkBlob(long id) { + return CVMLong.create(id).getEncoding(); + } + + // ── Key construction and round-trip ────────────────────────────────────── + + @Test + void testKeyLength() { + ABlob pk = pkBlob(1L); + ABlob key = HistoryKey.of(pk, 12345L); + // 1 byte length + pkLen bytes + 8 bytes nanotime + assertEquals(1 + pk.count() + 8, key.count()); + } + + @Test + void testExtractNanotime() { + ABlob pk = pkBlob(42L); + long nanotime = 0xDEADBEEFCAFEL; + ABlob key = HistoryKey.of(pk, nanotime); + assertEquals(nanotime, HistoryKey.extractNanotime(key)); + } + + @Test + void testNanotimeZero() { + ABlob pk = pkBlob(1L); + ABlob key = HistoryKey.of(pk, 0L); + assertEquals(0L, HistoryKey.extractNanotime(key)); + } + + @Test + void testNanotimeMaxLong() { + ABlob pk = pkBlob(1L); + ABlob key = HistoryKey.of(pk, Long.MAX_VALUE); + assertEquals(Long.MAX_VALUE, HistoryKey.extractNanotime(key)); + } + + @Test + void testDifferentPksDifferentKeys() { + ABlob key1 = HistoryKey.of(pkBlob(1L), 100L); + ABlob key2 = HistoryKey.of(pkBlob(2L), 100L); + assertFalse(key1.equals(key2)); + } + + @Test + void testSamePkDifferentNanotimeDifferentKeys() { + ABlob pk = pkBlob(1L); + ABlob key1 = HistoryKey.of(pk, 100L); + ABlob key2 = HistoryKey.of(pk, 200L); + assertFalse(key1.equals(key2)); + } + + @Test + void testSamePkSameNanotimeSameKey() { + ABlob pk = pkBlob(5L); + ABlob key1 = HistoryKey.of(pk, 999L); + ABlob key2 = HistoryKey.of(pk, 999L); + assertEquals(key1, key2); + } + + // ── Chronological ordering ──────────────────────────────────────────────── + + @Test + void testChronologicalOrdering() { + // Big-endian nanotime means lexicographic order == time order for same pk + ABlob pk = pkBlob(1L); + ABlob early = HistoryKey.of(pk, 1000L); + ABlob late = HistoryKey.of(pk, 2000L); + // Lexicographic comparison: early < late + assertTrue(early.compareTo(late) < 0); + } + + // ── Prefix matching ─────────────────────────────────────────────────────── + + @Test + void testPrefixLength() { + ABlob pk = pkBlob(1L); + ABlob prefix = HistoryKey.prefix(pk); + assertEquals(1 + pk.count(), prefix.count()); + } + + @Test + void testHasPrefixTrue() { + ABlob pk = pkBlob(7L); + ABlob key = HistoryKey.of(pk, 12345L); + ABlob prefix = HistoryKey.prefix(pk); + assertTrue(HistoryKey.hasPrefix(key, prefix)); + } + + @Test + void testHasPrefixFalseWrongPk() { + ABlob pk1 = pkBlob(1L); + ABlob pk2 = pkBlob(2L); + ABlob key = HistoryKey.of(pk1, 12345L); + ABlob prefix2 = HistoryKey.prefix(pk2); + assertFalse(HistoryKey.hasPrefix(key, prefix2)); + } + + @Test + void testHasPrefixFalseKeyTooShort() { + ABlob pk = pkBlob(1L); + ABlob prefix = HistoryKey.prefix(pk); + // A blob shorter than the prefix cannot match + ABlob shortBlob = Blob.wrap(new byte[]{0x01}); + assertFalse(HistoryKey.hasPrefix(shortBlob, prefix)); + } + + @Test + void testPkTooLongThrows() { + byte[] longPk = new byte[256]; // exceeds 255-byte limit + ABlob pk = Blob.wrap(longPk); + assertThrows(IllegalArgumentException.class, () -> HistoryKey.of(pk, 0L)); + } + + @Test + void testPk255BytesAccepted() { + byte[] maxPk = new byte[255]; + ABlob pk = Blob.wrap(maxPk); + ABlob key = HistoryKey.of(pk, 42L); + assertEquals(1 + 255 + 8, key.count()); + assertEquals(42L, HistoryKey.extractNanotime(key)); + } +} diff --git a/convex-db/src/test/java/convex/db/sql/BlockVecHeapBench.java b/convex-db/src/test/java/convex/db/sql/BlockVecHeapBench.java new file mode 100644 index 000000000..9d3cd888b --- /dev/null +++ b/convex-db/src/test/java/convex/db/sql/BlockVecHeapBench.java @@ -0,0 +1,97 @@ +package convex.db.sql; + +import java.util.ArrayList; +import java.util.List; + +import convex.core.crypto.AKeyPair; +import convex.core.data.ACell; +import convex.core.data.AVector; +import convex.core.data.Vectors; +import convex.core.data.prim.CVMLong; +import convex.db.calcite.ConvexTableEnumerator; +import convex.db.calcite.ConvexType; +import convex.db.lattice.SQLDatabase; +import convex.db.lattice.SQLSchema; +import convex.db.lattice.SQLTable; + +/** + * Verifies that the blockVec fast path in ConvexTableEnumerator + * is taken after a batch insert, and measures heap delta during scan. + * + * This is an in-memory test (no Etch) — it proves the code path works + * and measures blockVec vs Index scan overhead for in-memory data. + * The Etch benefit (Index trie reload from RefSoft) is larger and is + * measured by DBComparisonBench. + */ +public class BlockVecHeapBench { + + static final int[] SIZES = {10_000, 100_000}; + + public static void main(String[] args) throws Exception { + System.out.println("=== BlockVec Heap Bench ===\n"); + + for (int n : SIZES) { + System.out.printf("--- %,d rows ---%n", n); + + // --- Individual inserts (blockVec invalidated) --- + { + SQLDatabase db = SQLDatabase.create("bench", AKeyPair.generate()); + SQLSchema tables = db.tables(); + tables.createTable("t", + new String[]{"ID", "NM"}, + new ConvexType[]{ConvexType.INTEGER, ConvexType.VARCHAR}); + for (int i = 0; i < n; i++) { + tables.insert("t", Vectors.of(CVMLong.create(i), "Name-" + i)); + } + SQLTable tbl = tables.getLiveTable("t"); + AVector bv = tbl.getBlockVec(); + System.out.printf(" Individual inserts: blockVec=%s%n", + bv == null ? "null (Index path)" : "present (" + bv.count() + " blocks)"); + long h0 = usedHeap(); + int count = scanViaEnumerator(tables, "t"); + long h1 = usedHeap(); + System.out.printf(" Scan (Index path): %,d rows, heap delta: %+.1f MB%n", + count, (h1 - h0) / 1024.0 / 1024.0); + } + + // --- Batch insert (blockVec populated) --- + { + SQLDatabase db = SQLDatabase.create("bench", AKeyPair.generate()); + SQLSchema tables = db.tables(); + tables.createTable("t", + new String[]{"ID", "NM"}, + new ConvexType[]{ConvexType.INTEGER, ConvexType.VARCHAR}); + + List> rows = new ArrayList<>(n); + for (int i = 0; i < n; i++) { + rows.add(Vectors.of(CVMLong.create(i), "Name-" + i)); + } + tables.insertAll("t", rows); + SQLTable tbl = tables.getLiveTable("t"); + + AVector bv = tbl.getBlockVec(); + System.out.printf(" Batch insert: blockVec=%s%n", + bv == null ? "null" : "present (" + bv.count() + " blocks)"); + long h0 = usedHeap(); + int count = scanViaEnumerator(tables, "t"); + long h1 = usedHeap(); + System.out.printf(" Scan (blockVec path): %,d rows, heap delta: %+.1f MB%n", + count, (h1 - h0) / 1024.0 / 1024.0); + } + + System.out.println(); + } + } + + static int scanViaEnumerator(SQLSchema tables, String tableName) { + ConvexTableEnumerator e = new ConvexTableEnumerator(tables, tableName); + int count = 0; + while (e.moveNext()) count++; + return count; + } + + static long usedHeap() { + Runtime rt = Runtime.getRuntime(); + return rt.totalMemory() - rt.freeMemory(); + } +} diff --git a/convex-db/src/test/java/convex/db/sql/DBComparisonBench.java b/convex-db/src/test/java/convex/db/sql/DBComparisonBench.java new file mode 100644 index 000000000..c35e81fc6 --- /dev/null +++ b/convex-db/src/test/java/convex/db/sql/DBComparisonBench.java @@ -0,0 +1,2360 @@ +package convex.db.sql; + +import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.Statement; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.zip.GZIPOutputStream; + +import convex.core.data.ABlob; +import convex.core.data.ACell; +import convex.core.data.AVector; +import convex.core.data.Vectors; +import convex.core.data.prim.CVMLong; +import convex.db.ConvexDB; +import convex.db.calcite.ConvexType; +import convex.db.store.SegmentedEtchStore; +import convex.db.lattice.RowBlock; +import convex.db.lattice.SQLDatabase; +import convex.db.lattice.SQLRow; +import convex.db.lattice.SQLSchema; +import convex.db.lattice.SQLTable; +import convex.db.lattice.VersionedSQLSchema; +import convex.etch.EtchStore; +import convex.node.NodeServer; + +/** + * Comparison benchmark: Convex DB vs MariaDB. + * + *

Measures insert throughput, file size, and estimated compressed size for: + *

    + *
  • Convex JDBC PreparedStatement
    • + *
  • Convex JDBC PreparedStatement batch
    • + *
  • MariaDB JDBC PreparedStatement
    • + *
  • MariaDB JDBC PreparedStatement batch
    • + *
+ * + *

MariaDB connection is controlled by system properties: + * 

+ *   -Dmariadb.user=root   (default: root)
+ *   -Dmariadb.pass=       (default: empty)
+ *   -Dmariadb.host=localhost (default: localhost)
+ *   -Dmariadb.port=3306   (default: 3306)
+ *
+ * + *

Etch files are written to STORE_DIR. + */ +public class DBComparisonBench { + + static final int ROW_COUNT = 100_000; + static final int BATCH_SIZE = 1_000; + static final int SINGLE_WARMUP = 200; // inserts before timing single-row + static final int SINGLE_SAMPLES = 1_000; // timed single-row inserts to average + static final int RW_OPS = 10_000; // lookup/update ops per read-write bench + static final int[] REPLICATION_K = {100, 1_000, 10_000}; // changes applied on node B + static final String DB_NAME = "bench"; + static final String TABLE_NAME = "t"; + static final String TABLE_NAME_MEM = "t_mem"; + static final String STORE_DIR = "/home/rob/etch/"; + static final Random RND = new Random(42); + + // MariaDB connection params (overridable via -Dmariadb.* system properties) + static final String MARIADB_HOST = System.getProperty("mariadb.host", "localhost"); + static final String MARIADB_PORT = System.getProperty("mariadb.port", "3306"); + static final String MARIADB_USER = System.getProperty("mariadb.user", "tempuser"); + static final String MARIADB_PASS = System.getProperty("mariadb.pass", "123456"); + static final String MARIADB_DB = "temp"; + static final String MARIADB_URL = + "jdbc:mariadb://" + MARIADB_HOST + ":" + MARIADB_PORT + "/" + MARIADB_DB + + "?rewriteBatchedStatements=true&useServerPrepStmts=true"; + + // PostgreSQL connection params (overridable via -Dpsql.* system properties) + static final String PSQL_HOST = System.getProperty("psql.host", "localhost"); + static final String PSQL_PORT = System.getProperty("psql.port", "5432"); + static final String PSQL_USER = System.getProperty("psql.user", "tempuser"); + static final String PSQL_PASS = System.getProperty("psql.pass", "123456"); + static final String PSQL_DB = System.getProperty("psql.db", "temp"); + static final String PSQL_URL = "jdbc:postgresql://" + PSQL_HOST + ":" + PSQL_PORT + "/" + PSQL_DB; + + // ── Result record ────────────────────────────────────────────────────────── + + record BenchResult( + String label, + int rows, + long elapsedNs, + long writtenBytes, // actual data written (etchDataBytes for Etch, information_schema for MariaDB) + long allocatedBytes, // OS-level file allocation (same as writtenBytes for MariaDB; storeFile.length() for Etch) + long compressedBytes, // estimated gz size of writtenBytes (-1 if not measured) + long heapDeltaBytes, + long dbRowsBefore, // live rows in DB before benchmark (-1 if not measured) + long dbRowsAfter // live rows in DB after benchmark (-1 if not measured) + ) { + // Backward-compatible constructor for benchmarks that don't measure row counts + BenchResult(String label, int rows, long elapsedNs, + long writtenBytes, long allocatedBytes, long compressedBytes, long heapDeltaBytes) { + this(label, rows, elapsedNs, writtenBytes, allocatedBytes, compressedBytes, heapDeltaBytes, -1, -1); + } + double rowsPerSec() { return rows / (elapsedNs / 1e9); } + double writtenPerRow() { return writtenBytes > 0 ? (double) writtenBytes / rows : -1; } + double allocatedPerRow() { return allocatedBytes > 0 ? (double) allocatedBytes / rows : -1; } + double compPerRow() { return compressedBytes > 0 ? (double) compressedBytes / rows : -1; } + double writtenMB() { return writtenBytes > 0 ? writtenBytes / (1024.0 * 1024.0) : -1; } + double allocatedMB() { return allocatedBytes > 0 ? allocatedBytes / (1024.0 * 1024.0) : -1; } + double heapDeltaMB() { return heapDeltaBytes / (1024.0 * 1024.0); } + } + + // ── Replication result record ───────────────────────────────────────────── + + record ReplicationResult( + String db, + int k, // changes applied on node B + long payloadBytes, // bytes that need to be transferred for sync + long fullTableBytes, // full table size (= naive sync cost without dedup) + String payloadNote // how payload was measured + ) { + double payloadKB() { return payloadBytes / 1024.0; } + double bytesPerChange() { return (double) payloadBytes / k; } + double fullTableMB() { return fullTableBytes / (1024.0 * 1024.0); } + double payloadPct() { return fullTableBytes > 0 ? 100.0 * payloadBytes / fullTableBytes : -1; } + } + + // ── Entry point ──────────────────────────────────────────────────────────── + + public static void main(String[] args) throws Exception { + // ── Compare mode ──────────────────────────────────────────────────── + if (args.length == 3 && "compare".equals(args[0])) { + compareResultFiles(new File(args[1]), new File(args[2])); + return; + } + + System.out.println("=== Convex DB vs MariaDB / PostgreSQL — Insert Benchmark ==="); + System.out.printf("Rows: %,d | Batch size: %,d | Etch dir: %s%n%n", ROW_COUNT, BATCH_SIZE, STORE_DIR); + + List results = new ArrayList<>(); + + boolean mariaAvailable = checkMariaDB(); + if (mariaAvailable) { + results.add(benchMariaDBPrepared()); + System.gc(); + results.add(benchMariaDBBatch()); + System.gc(); + results.add(benchMariaDBLoadFile()); + System.gc(); + results.add(benchMariaDBVersionedPrepared()); + System.gc(); + results.add(benchMariaDBVersionedBatch()); + System.gc(); + results.add(benchMariaDBVersionedLoadFile()); + System.gc(); + results.add(benchMariaDBMemoryPrepared()); + System.gc(); + results.add(benchMariaDBMemoryBatch()); + System.gc(); + } else { + System.out.println("MariaDB not available — skipping MariaDB benchmarks."); + System.out.printf(" Connect string: %s user=%s%n%n", MARIADB_URL, MARIADB_USER); + } + boolean psqlAvailable = checkPostgres(); + if (psqlAvailable) { + results.add(benchPostgresPrepared()); + System.gc(); + results.add(benchPostgresBatch()); + System.gc(); + results.add(benchPostgresCopy()); + System.gc(); + } else { + System.out.println("PostgreSQL not available — skipping PostgreSQL benchmarks."); + System.out.printf(" Connect string: %s user=%s%n%n", PSQL_URL, PSQL_USER); + } + results.add(benchConvexDirect()); + System.gc(); + results.add(benchConvexDirectVersioned()); + System.gc(); + results.add(benchConvexPrepared()); + System.gc(); + results.add(benchConvexBatch()); + System.gc(); + + printComparison(results); + + // ── Single-row latency ────────────────────────────────────────────── + System.out.println("\n=== Single-row insert latency ==="); + System.out.printf("Warmup: %,d | Samples: %,d%n%n", SINGLE_WARMUP, SINGLE_SAMPLES); + + List singleResults = new ArrayList<>(); + singleResults.add(benchConvexDirectSingle()); + System.gc(); + singleResults.add(benchConvexPreparedSingle()); + System.gc(); + if (mariaAvailable) { + singleResults.add(benchMariaDBPreparedSingle()); + System.gc(); + singleResults.add(benchMariaDBLoadFileSingle()); + System.gc(); + singleResults.add(benchMariaDBVersionedPreparedSingle()); + System.gc(); + singleResults.add(benchMariaDBMemoryPreparedSingle()); + System.gc(); + } + if (psqlAvailable) { + singleResults.add(benchPostgresPreparedSingle()); + System.gc(); + } + printSingleRowComparison(singleResults); + + // ── Read / Update benchmarks ──────────────────────────────────────── + System.out.printf("%n=== Read / Update benchmarks (ops: %,d on %,d-row dataset) ===%n%n", RW_OPS, ROW_COUNT); + + List rwResults = new ArrayList<>(); + rwResults.addAll(benchConvexDirectRW()); + System.gc(); + rwResults.addAll(benchConvexJdbcRW()); + System.gc(); + if (mariaAvailable) { + rwResults.addAll(benchMariaDBRW(false)); + System.gc(); + rwResults.addAll(benchMariaDBRW(true)); + System.gc(); + rwResults.addAll(benchMariaDBMemoryRW()); + System.gc(); + } + if (psqlAvailable) { + rwResults.addAll(benchPostgresRW()); + System.gc(); + } + printRWComparison(rwResults); + + // ── Full scan benchmarks ──────────────────────────────────────────── + System.out.println("\n=== Full scan (SELECT * heap cost) ==="); + List scanResults = new ArrayList<>(); + scanResults.add(benchConvexJdbcScan()); + System.gc(); + if (mariaAvailable) { + scanResults.add(benchMariaDBScan()); + System.gc(); + scanResults.add(benchMariaDBMemoryScan()); + System.gc(); + } + printScanComparison(scanResults); + + // ── Replication / Dedup benchmark ─────────────────────────────────── + //benchmarkReplication(mariaAvailable, psqlAvailable); + + // ── Merge latency benchmark ────────────────────────────────────────── + System.out.println("\n=== Merge latency benchmark ==="); + List mergeResults = benchConvexMergeLatency(); + System.gc(); + + // ── Save all results ───────────────────────────────────────────────── + List all = new ArrayList<>(); + all.addAll(results); + all.addAll(singleResults); + all.addAll(rwResults); + all.addAll(scanResults); + all.addAll(mergeResults); + File saved = saveResults(all); + System.out.println("\nResults saved to: " + saved.getPath()); + autoCompare(saved); + System.out.println("\nTo compare with a specific baseline run:"); + System.out.printf(" java DBComparisonBench compare %s%n", saved.getName()); + + System.out.println("\nDone."); + } + + // ── Convex benchmarks ────────────────────────────────────────────────────── + + static BenchResult benchConvexDirect() throws Exception { + System.out.println("--- Convex direct lattice insert (plain, no history) ---"); + File storeDir = new File(STORE_DIR, "cmp-convex-direct"); + SegmentedEtchStore store = SegmentedEtchStore.createFresh(storeDir); + NodeServer server = ConvexDB.createNodeServer(store); + server.launch(); + SQLDatabase db = SQLDatabase.connect(server.getCursor(), DB_NAME); + SQLSchema schema = db.tables(); // plain — matches JDBC benchmarks (no history overhead) + createConvexTable(schema); + + long dbRowsBefore = countConvexRows(schema); + long heapBefore = usedHeap(); + long start = System.nanoTime(); + + List> batch = new ArrayList<>(ROW_COUNT); + for (int i = 0; i < ROW_COUNT; i++) + batch.add(Vectors.of(CVMLong.create(i), "LEID-" + i, "Name-" + i, CVMLong.create(RND.nextLong()))); + schema.insertAll(TABLE_NAME, batch); + batch = null; // release input batch — Index now holds compact Blobs; measure only those + System.gc(); + + long elapsed = System.nanoTime() - start; + long heapDelta = usedHeap() - heapBefore; + long dbRowsAfter = countConvexRows(schema); + + server.persistSnapshot(server.getLocalValue()); + long writtenBytes = etchDataBytes(store.getHotFile()); + long allocBytes = store.getHotFile().length(); + long compBytes = estimateCompressedSize(store.getHotFile()); + System.out.printf(" Elapsed: %,.0f ms | Written: %.1f MB | Alloc: %.1f MB | ~gz: %.1f MB | DB rows: %,d→%,d%n", + elapsed / 1e6, writtenBytes / (1024.0 * 1024.0), allocBytes / (1024.0 * 1024.0), compBytes / (1024.0 * 1024.0), + dbRowsBefore, dbRowsAfter); + + server.close(); + store.close(); + return new BenchResult("Convex direct", ROW_COUNT, elapsed, writtenBytes, allocBytes, compBytes, heapDelta, dbRowsBefore, dbRowsAfter); + } + + static BenchResult benchConvexDirectVersioned() throws Exception { + System.out.println("--- Convex direct lattice insert (versioned, with history) ---"); + File storeDir = new File(STORE_DIR, "cmp-convex-direct-versioned"); + SegmentedEtchStore store = SegmentedEtchStore.createFresh(storeDir); + NodeServer server = ConvexDB.createNodeServer(store); + server.launch(); + SQLDatabase db = SQLDatabase.connect(server.getCursor(), DB_NAME); + VersionedSQLSchema schema = VersionedSQLSchema.wrap(db.tables()); + createConvexTable(schema); + + long dbRowsBefore = countConvexRows(schema); + long heapBefore = usedHeap(); + long start = System.nanoTime(); + + List> batch = new ArrayList<>(ROW_COUNT); + for (int i = 0; i < ROW_COUNT; i++) + batch.add(Vectors.of(CVMLong.create(i), "LEID-" + i, "Name-" + i, CVMLong.create(RND.nextLong()))); + schema.insertAll(TABLE_NAME, batch); + batch = null; // release input batch before measuring Index-only heap + System.gc(); + + long elapsed = System.nanoTime() - start; + long heapDelta = usedHeap() - heapBefore; + long dbRowsAfter = countConvexRows(schema); + + server.persistSnapshot(server.getLocalValue()); + long writtenBytes = etchDataBytes(store.getHotFile()); + long allocBytes = store.getHotFile().length(); + long compBytes = estimateCompressedSize(store.getHotFile()); + System.out.printf(" Elapsed: %,.0f ms | Written: %.1f MB | Alloc: %.1f MB | ~gz: %.1f MB | DB rows: %,d→%,d%n", + elapsed / 1e6, writtenBytes / (1024.0 * 1024.0), allocBytes / (1024.0 * 1024.0), compBytes / (1024.0 * 1024.0), + dbRowsBefore, dbRowsAfter); + + server.close(); + store.close(); + return new BenchResult("Convex direct versioned", ROW_COUNT, elapsed, writtenBytes, allocBytes, compBytes, heapDelta, dbRowsBefore, dbRowsAfter); + } + + static BenchResult benchConvexPrepared() throws Exception { + System.out.println("--- Convex JDBC PreparedStatement ---"); + File storeDir = new File(STORE_DIR, "cmp-convex-prepared"); + SegmentedEtchStore store = SegmentedEtchStore.createFresh(storeDir); + NodeServer server = ConvexDB.createNodeServer(store); + server.launch(); + ConvexDB cdb = ConvexDB.connect(server.getCursor()); + SQLDatabase db = cdb.database(DB_NAME); + SQLSchema schema = db.tables(); + createConvexTable(schema); + cdb.register(DB_NAME); + + long dbRowsBefore = countConvexRows(schema); + long heapBefore = usedHeap(); + long start = System.nanoTime(); + + try (Connection conn = DriverManager.getConnection("jdbc:convex:database=" + DB_NAME); + PreparedStatement ps = conn.prepareStatement("INSERT INTO " + TABLE_NAME + " VALUES (?, ?, ?, ?)")) { + + for (int i = 0; i < ROW_COUNT; i++) { + ps.setInt(1, i); + ps.setString(2, "LEID-" + i); + ps.setString(3, "Name-" + i); + ps.setLong(4, RND.nextLong()); + ps.executeUpdate(); + if ((i + 1) % 10_000 == 0) + System.out.printf(" %,d rows%n", i + 1); + } + } + + long elapsed = System.nanoTime() - start; + System.gc(); + long heapDelta = usedHeap() - heapBefore; + long dbRowsAfter = countConvexRows(schema); + + server.persistSnapshot(server.getLocalValue()); + long writtenBytes = etchDataBytes(store.getHotFile()); + long allocBytes = store.getHotFile().length(); + long compBytes = estimateCompressedSize(store.getHotFile()); + System.out.printf(" Elapsed: %,.0f ms | Written: %.1f MB | Alloc: %.1f MB | ~gz: %.1f MB | DB rows: %,d→%,d%n", + elapsed / 1e6, writtenBytes / (1024.0 * 1024.0), allocBytes / (1024.0 * 1024.0), compBytes / (1024.0 * 1024.0), + dbRowsBefore, dbRowsAfter); + + cdb.unregister(DB_NAME); + server.close(); + store.close(); + return new BenchResult("Convex JDBC prepared", ROW_COUNT, elapsed, writtenBytes, allocBytes, compBytes, heapDelta, dbRowsBefore, dbRowsAfter); + } + + static BenchResult benchConvexBatch() throws Exception { + System.out.println("--- Convex JDBC PreparedStatement batch ---"); + File storeDir = new File(STORE_DIR, "cmp-convex-batch"); + SegmentedEtchStore store = SegmentedEtchStore.createFresh(storeDir); + NodeServer server = ConvexDB.createNodeServer(store); + server.launch(); + ConvexDB cdb = ConvexDB.connect(server.getCursor()); + SQLDatabase db = cdb.database(DB_NAME); + SQLSchema schema = db.tables(); + createConvexTable(schema); + cdb.register(DB_NAME); + + long dbRowsBefore = countConvexRows(schema); + long heapBefore = usedHeap(); + long start = System.nanoTime(); + + try (Connection conn = DriverManager.getConnection("jdbc:convex:database=" + DB_NAME); + PreparedStatement ps = conn.prepareStatement("INSERT INTO " + TABLE_NAME + " VALUES (?, ?, ?, ?)")) { + + for (int i = 0; i < ROW_COUNT; i++) { + ps.setInt(1, i); + ps.setString(2, "LEID-" + i); + ps.setString(3, "Name-" + i); + ps.setLong(4, RND.nextLong()); + ps.addBatch(); + if ((i + 1) % BATCH_SIZE == 0) { + ps.executeBatch(); + if ((i + 1) % 10_000 == 0) + System.out.printf(" %,d rows%n", i + 1); + } + } + // flush remaining + ps.executeBatch(); + } + + long elapsed = System.nanoTime() - start; + System.gc(); + long heapDelta = usedHeap() - heapBefore; + long dbRowsAfter = countConvexRows(schema); + + server.persistSnapshot(server.getLocalValue()); + long writtenBytes = etchDataBytes(store.getHotFile()); + long allocBytes = store.getHotFile().length(); + long compBytes = estimateCompressedSize(store.getHotFile()); + System.out.printf(" Elapsed: %,.0f ms | Written: %.1f MB | Alloc: %.1f MB | ~gz: %.1f MB | DB rows: %,d→%,d%n", + elapsed / 1e6, writtenBytes / (1024.0 * 1024.0), allocBytes / (1024.0 * 1024.0), compBytes / (1024.0 * 1024.0), + dbRowsBefore, dbRowsAfter); + + cdb.unregister(DB_NAME); + server.close(); + store.close(); + return new BenchResult("Convex JDBC batch", ROW_COUNT, elapsed, writtenBytes, allocBytes, compBytes, heapDelta, dbRowsBefore, dbRowsAfter); + } + + // ── MariaDB benchmarks ───────────────────────────────────────────────────── + + static boolean checkMariaDB() { + try { + setupMariaDB(); + return true; + } catch (Exception e) { + System.out.println(" MariaDB setup failed: " + e.getMessage()); + return false; + } + } + + static void setupMariaDB() throws Exception { + // Connect directly to the existing database — no CREATE DATABASE needed + String connUrl = MARIADB_URL + "&user=" + MARIADB_USER + "&password=" + MARIADB_PASS; + try (Connection conn = DriverManager.getConnection(connUrl); + Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP TABLE IF EXISTS " + TABLE_NAME); + stmt.executeUpdate( + "CREATE TABLE " + TABLE_NAME + " (" + + "id BIGINT PRIMARY KEY, " + + "leid VARCHAR(32), " + + "nm VARCHAR(32), " + + "rnd BIGINT" + + ") ENGINE=InnoDB ROW_FORMAT=COMPRESSED"); + } + } + + static BenchResult benchMariaDBPrepared() throws Exception { + System.out.println("--- MariaDB JDBC PreparedStatement ---"); + setupMariaDB(); + + String connUrl = MARIADB_URL + "&user=" + MARIADB_USER + "&password=" + MARIADB_PASS; + long dbRowsBefore = countMariaDBRows(); + long heapBefore = usedHeap(); + long start = System.nanoTime(); + + try (Connection conn = DriverManager.getConnection(connUrl); + PreparedStatement ps = conn.prepareStatement( + "INSERT INTO " + TABLE_NAME + " (id,leid,nm,rnd) VALUES (?,?,?,?) " + + "ON DUPLICATE KEY UPDATE leid=VALUES(leid), nm=VALUES(nm), rnd=VALUES(rnd)")) { + + for (int i = 0; i < ROW_COUNT; i++) { + ps.setInt(1, i); + ps.setString(2, "LEID-" + i); + ps.setString(3, "Name-" + i); + ps.setLong(4, RND.nextLong()); + ps.executeUpdate(); + if ((i + 1) % 10_000 == 0) + System.out.printf(" %,d rows%n", i + 1); + } + } + + long elapsed = System.nanoTime() - start; + long heapDelta = usedHeap() - heapBefore; + long dbRowsAfter = countMariaDBRows(); + long fileBytes = mariaDBTableBytes(); + System.out.printf(" Elapsed: %,.0f ms | Table size: %.1f MB (InnoDB compressed) | DB rows: %,d→%,d%n", + elapsed / 1e6, fileBytes / (1024.0 * 1024.0), dbRowsBefore, dbRowsAfter); + + return new BenchResult("MariaDB JDBC prepared", ROW_COUNT, elapsed, fileBytes, fileBytes, fileBytes, heapDelta, dbRowsBefore, dbRowsAfter); + } + + static BenchResult benchMariaDBBatch() throws Exception { + System.out.println("--- MariaDB JDBC PreparedStatement batch ---"); + setupMariaDB(); + + String connUrl = MARIADB_URL + "&user=" + MARIADB_USER + "&password=" + MARIADB_PASS; + long dbRowsBefore = countMariaDBRows(); + long heapBefore = usedHeap(); + long start = System.nanoTime(); + + try (Connection conn = DriverManager.getConnection(connUrl); + PreparedStatement ps = conn.prepareStatement( + "INSERT INTO " + TABLE_NAME + " (id,leid,nm,rnd) VALUES (?,?,?,?) " + + "ON DUPLICATE KEY UPDATE leid=VALUES(leid), nm=VALUES(nm), rnd=VALUES(rnd)")) { + + for (int i = 0; i < ROW_COUNT; i++) { + ps.setInt(1, i); + ps.setString(2, "LEID-" + i); + ps.setString(3, "Name-" + i); + ps.setLong(4, RND.nextLong()); + ps.addBatch(); + if ((i + 1) % BATCH_SIZE == 0) { + ps.executeBatch(); + if ((i + 1) % 10_000 == 0) + System.out.printf(" %,d rows%n", i + 1); + } + } + ps.executeBatch(); + } + + long elapsed = System.nanoTime() - start; + long heapDelta = usedHeap() - heapBefore; + long dbRowsAfter = countMariaDBRows(); + long fileBytes = mariaDBTableBytes(); + System.out.printf(" Elapsed: %,.0f ms | Table size: %.1f MB (InnoDB compressed) | DB rows: %,d→%,d%n", + elapsed / 1e6, fileBytes / (1024.0 * 1024.0), dbRowsBefore, dbRowsAfter); + + return new BenchResult("MariaDB JDBC batch", ROW_COUNT, elapsed, fileBytes, fileBytes, fileBytes, heapDelta, dbRowsBefore, dbRowsAfter); + } + + static BenchResult benchMariaDBLoadFile() throws Exception { + System.out.println("--- MariaDB LOAD DATA INFILE (CSV) ---"); + setupMariaDB(); + + // Phase 1: write CSV file + File csvFile = File.createTempFile("convex_bench_", ".csv"); + csvFile.deleteOnExit(); + long writeStart = System.nanoTime(); + try (BufferedWriter bw = new BufferedWriter(new FileWriter(csvFile))) { + for (int i = 0; i < ROW_COUNT; i++) { + bw.write(i + "\tLEID-" + i + "\tName-" + i + "\t" + RND.nextLong() + "\n"); + } + } + long writeNs = System.nanoTime() - writeStart; + System.out.printf(" CSV written: %s (%.1f MB, %,.0f ms)%n", + csvFile.getName(), csvFile.length() / (1024.0 * 1024.0), writeNs / 1e6); + + // Phase 2: LOAD DATA INFILE + String connUrl = MARIADB_URL + "&user=" + MARIADB_USER + "&password=" + MARIADB_PASS + + "&allowLocalInfile=true"; + long dbRowsBefore = countMariaDBRows(); + long heapBefore = usedHeap(); + long loadStart = System.nanoTime(); + + try (Connection conn = DriverManager.getConnection(connUrl); + Statement stmt = conn.createStatement()) { + stmt.execute( + "LOAD DATA LOCAL INFILE '" + csvFile.getAbsolutePath().replace("\\", "/") + "' " + + "INTO TABLE " + TABLE_NAME + " " + + "FIELDS TERMINATED BY '\\t' " + + "LINES TERMINATED BY '\\n' " + + "(id, leid, nm, rnd)"); + } + + long loadNs = System.nanoTime() - loadStart; + long totalNs = writeNs + loadNs; + long heapDelta = usedHeap() - heapBefore; + long dbRowsAfter = countMariaDBRows(); + long fileBytes = mariaDBTableBytes(); + System.out.printf(" Load: %,.0f ms | Total (write+load): %,.0f ms | Table: %.1f MB | DB rows: %,d→%,d%n", + loadNs / 1e6, totalNs / 1e6, fileBytes / (1024.0 * 1024.0), dbRowsBefore, dbRowsAfter); + + csvFile.delete(); + // Report total elapsed (write + load) so comparison is apples-to-apples + return new BenchResult("MariaDB LOAD FILE", ROW_COUNT, totalNs, fileBytes, fileBytes, fileBytes, heapDelta, dbRowsBefore, dbRowsAfter); + } + + // ── PostgreSQL benchmarks ───────────────────────────────────────────────── + + static boolean checkPostgres() { + try { + setupPostgres(); + return true; + } catch (Exception e) { + System.out.println(" PostgreSQL setup failed: " + e.getMessage()); + return false; + } + } + + static void setupPostgres() throws Exception { + try (Connection conn = DriverManager.getConnection(PSQL_URL, PSQL_USER, PSQL_PASS); + Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP TABLE IF EXISTS " + TABLE_NAME); + stmt.executeUpdate( + "CREATE TABLE " + TABLE_NAME + " (" + + "id BIGINT PRIMARY KEY, " + + "leid VARCHAR(32), " + + "nm VARCHAR(32), " + + "rnd BIGINT)"); + } + } + + static long postgresTableBytes() throws Exception { + try (Connection conn = DriverManager.getConnection(PSQL_URL, PSQL_USER, PSQL_PASS); + Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("SELECT pg_total_relation_size('" + TABLE_NAME + "') AS b"); + if (rs.next()) return rs.getLong("b"); + } + return -1; + } + + static BenchResult benchPostgresPrepared() throws Exception { + System.out.println("--- PostgreSQL JDBC PreparedStatement ---"); + setupPostgres(); + + long dbRowsBefore = countPostgresRows(); + long heapBefore = usedHeap(); + long start = System.nanoTime(); + + try (Connection conn = DriverManager.getConnection(PSQL_URL, PSQL_USER, PSQL_PASS); + PreparedStatement ps = conn.prepareStatement( + "INSERT INTO " + TABLE_NAME + " (id,leid,nm,rnd) VALUES (?,?,?,?) " + + "ON CONFLICT (id) DO UPDATE SET leid=EXCLUDED.leid, nm=EXCLUDED.nm, rnd=EXCLUDED.rnd")) { + + for (int i = 0; i < ROW_COUNT; i++) { + ps.setInt(1, i); ps.setString(2, "LEID-" + i); ps.setString(3, "Name-" + i); ps.setLong(4, RND.nextLong()); + ps.executeUpdate(); + if ((i + 1) % 10_000 == 0) System.out.printf(" %,d rows%n", i + 1); + } + } + + long elapsed = System.nanoTime() - start; + long heapDelta = usedHeap() - heapBefore; + long dbRowsAfter = countPostgresRows(); + long fileBytes = postgresTableBytes(); + System.out.printf(" Elapsed: %,.0f ms | Table: %.1f MB | DB rows: %,d→%,d%n", + elapsed / 1e6, fileBytes / (1024.0 * 1024.0), dbRowsBefore, dbRowsAfter); + return new BenchResult("PostgreSQL prepared", ROW_COUNT, elapsed, fileBytes, fileBytes, -1, heapDelta, dbRowsBefore, dbRowsAfter); + } + + static BenchResult benchPostgresBatch() throws Exception { + System.out.println("--- PostgreSQL JDBC PreparedStatement batch ---"); + setupPostgres(); + + long dbRowsBefore = countPostgresRows(); + long heapBefore = usedHeap(); + long start = System.nanoTime(); + + try (Connection conn = DriverManager.getConnection(PSQL_URL, PSQL_USER, PSQL_PASS); + PreparedStatement ps = conn.prepareStatement( + "INSERT INTO " + TABLE_NAME + " (id,leid,nm,rnd) VALUES (?,?,?,?) " + + "ON CONFLICT (id) DO UPDATE SET leid=EXCLUDED.leid, nm=EXCLUDED.nm, rnd=EXCLUDED.rnd")) { + + for (int i = 0; i < ROW_COUNT; i++) { + ps.setInt(1, i); ps.setString(2, "LEID-" + i); ps.setString(3, "Name-" + i); ps.setLong(4, RND.nextLong()); + ps.addBatch(); + if ((i + 1) % BATCH_SIZE == 0) { + ps.executeBatch(); + if ((i + 1) % 10_000 == 0) System.out.printf(" %,d rows%n", i + 1); + } + } + ps.executeBatch(); + } + + long elapsed = System.nanoTime() - start; + long heapDelta = usedHeap() - heapBefore; + long dbRowsAfter = countPostgresRows(); + long fileBytes = postgresTableBytes(); + System.out.printf(" Elapsed: %,.0f ms | Table: %.1f MB | DB rows: %,d→%,d%n", + elapsed / 1e6, fileBytes / (1024.0 * 1024.0), dbRowsBefore, dbRowsAfter); + return new BenchResult("PostgreSQL batch", ROW_COUNT, elapsed, fileBytes, fileBytes, -1, heapDelta, dbRowsBefore, dbRowsAfter); + } + + static BenchResult benchPostgresCopy() throws Exception { + System.out.println("--- PostgreSQL COPY FROM STDIN (CSV) ---"); + setupPostgres(); + + // Phase 1: write CSV file + File csvFile = File.createTempFile("convex_psql_", ".csv"); + csvFile.deleteOnExit(); + long writeStart = System.nanoTime(); + try (BufferedWriter bw = new BufferedWriter(new FileWriter(csvFile))) { + for (int i = 0; i < ROW_COUNT; i++) + bw.write(i + "\tLEID-" + i + "\tName-" + i + "\t" + RND.nextLong() + "\n"); + } + long writeNs = System.nanoTime() - writeStart; + System.out.printf(" CSV written: %.1f MB, %,.0f ms%n", csvFile.length() / (1024.0 * 1024.0), writeNs / 1e6); + + // Phase 2: COPY FROM STDIN via CopyManager + long dbRowsBefore = countPostgresRows(); + long heapBefore = usedHeap(); + long copyStart = System.nanoTime(); + + try (Connection conn = DriverManager.getConnection(PSQL_URL, PSQL_USER, PSQL_PASS); + FileReader fr = new FileReader(csvFile)) { + org.postgresql.copy.CopyManager cm = + conn.unwrap(org.postgresql.PGConnection.class).getCopyAPI(); + cm.copyIn("COPY " + TABLE_NAME + " (id,leid,nm,rnd) FROM STDIN", fr); + } + + long copyNs = System.nanoTime() - copyStart; + long totalNs = writeNs + copyNs; + long heapDelta = usedHeap() - heapBefore; + long dbRowsAfter = countPostgresRows(); + long fileBytes = postgresTableBytes(); + System.out.printf(" COPY: %,.0f ms | Total: %,.0f ms | Table: %.1f MB | DB rows: %,d→%,d%n", + copyNs / 1e6, totalNs / 1e6, fileBytes / (1024.0 * 1024.0), dbRowsBefore, dbRowsAfter); + + csvFile.delete(); + return new BenchResult("PostgreSQL COPY", ROW_COUNT, totalNs, fileBytes, fileBytes, -1, heapDelta, dbRowsBefore, dbRowsAfter); + } + + static BenchResult benchPostgresPreparedSingle() throws Exception { + System.out.println("--- Single-row: PostgreSQL JDBC PreparedStatement ---"); + setupPostgres(); + + try (Connection conn = DriverManager.getConnection(PSQL_URL, PSQL_USER, PSQL_PASS); + PreparedStatement ps = conn.prepareStatement( + "INSERT INTO " + TABLE_NAME + " (id,leid,nm,rnd) VALUES (?,?,?,?) " + + "ON CONFLICT (id) DO UPDATE SET leid=EXCLUDED.leid, nm=EXCLUDED.nm, rnd=EXCLUDED.rnd")) { + + for (int i = 0; i < SINGLE_WARMUP; i++) { + ps.setInt(1, i); ps.setString(2, "LEID-" + i); ps.setString(3, "Name-" + i); ps.setLong(4, RND.nextLong()); + ps.executeUpdate(); + } + long start = System.nanoTime(); + for (int i = SINGLE_WARMUP; i < SINGLE_WARMUP + SINGLE_SAMPLES; i++) { + ps.setInt(1, i); ps.setString(2, "LEID-" + i); ps.setString(3, "Name-" + i); ps.setLong(4, RND.nextLong()); + ps.executeUpdate(); + } + long elapsed = System.nanoTime() - start; + System.out.printf(" Avg latency: %.1f µs%n", elapsed / 1000.0 / SINGLE_SAMPLES); + return new BenchResult("PostgreSQL (1-row)", SINGLE_SAMPLES, elapsed, -1, -1, -1, 0); + } + } + + static List benchPostgresRW() throws Exception { + System.out.println("--- Read/Update: PostgreSQL ---"); + setupPostgres(); + + // Load via batch + try (Connection conn = DriverManager.getConnection(PSQL_URL, PSQL_USER, PSQL_PASS); + PreparedStatement ins = conn.prepareStatement( + "INSERT INTO " + TABLE_NAME + " (id,leid,nm,rnd) VALUES (?,?,?,?) " + + "ON CONFLICT (id) DO UPDATE SET leid=EXCLUDED.leid, nm=EXCLUDED.nm, rnd=EXCLUDED.rnd")) { + for (int i = 0; i < ROW_COUNT; i++) { + ins.setInt(1, i); ins.setString(2, "LEID-" + i); ins.setString(3, "Name-" + i); ins.setLong(4, RND.nextLong()); + ins.addBatch(); + if ((i + 1) % BATCH_SIZE == 0) ins.executeBatch(); + } + ins.executeBatch(); + } + System.out.printf(" Loaded %,d rows%n", ROW_COUNT); + + try (Connection conn = DriverManager.getConnection(PSQL_URL, PSQL_USER, PSQL_PASS); + PreparedStatement sel = conn.prepareStatement("SELECT id,leid,nm,rnd FROM " + TABLE_NAME + " WHERE id=?"); + PreparedStatement upd = conn.prepareStatement("UPDATE " + TABLE_NAME + " SET leid=?,nm=?,rnd=? WHERE id=?")) { + + long t0 = System.nanoTime(); + for (int i = 0; i < RW_OPS; i++) { + sel.setLong(1, Math.abs(RND.nextLong()) % ROW_COUNT); + sel.executeQuery().close(); + } + long lookupNs = System.nanoTime() - t0; + System.out.printf(" Lookup avg: %.1f µs%n", lookupNs / 1000.0 / RW_OPS); + + long t1 = System.nanoTime(); + for (int i = 0; i < RW_OPS; i++) { + long id = Math.abs(RND.nextLong()) % ROW_COUNT; + upd.setString(1, "LEID-" + id); upd.setString(2, "Name-" + id); upd.setLong(3, RND.nextLong()); upd.setLong(4, id); + upd.executeUpdate(); + } + long updateNs = System.nanoTime() - t1; + System.out.printf(" Update avg: %.1f µs%n", updateNs / 1000.0 / RW_OPS); + + return List.of( + new BenchResult("PostgreSQL lookup", RW_OPS, lookupNs, -1, -1, -1, 0), + new BenchResult("PostgreSQL update", RW_OPS, updateNs, -1, -1, -1, 0)); + } + } + + // ── MariaDB system-versioned benchmarks ──────────────────────────────────── + + static void setupMariaDBVersioned() throws Exception { + String connUrl = MARIADB_URL + "&user=" + MARIADB_USER + "&password=" + MARIADB_PASS; + try (Connection conn = DriverManager.getConnection(connUrl); + Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP TABLE IF EXISTS " + TABLE_NAME); + stmt.executeUpdate( + "CREATE TABLE " + TABLE_NAME + " (" + + "id BIGINT PRIMARY KEY, " + + "leid VARCHAR(32), " + + "nm VARCHAR(32), " + + "rnd BIGINT" + + ") ENGINE=InnoDB ROW_FORMAT=COMPRESSED WITH SYSTEM VERSIONING"); + } + } + + static BenchResult benchMariaDBVersionedPrepared() throws Exception { + System.out.println("--- MariaDB versioned JDBC PreparedStatement ---"); + setupMariaDBVersioned(); + + String connUrl = MARIADB_URL + "&user=" + MARIADB_USER + "&password=" + MARIADB_PASS; + long dbRowsBefore = countMariaDBRows(); + long heapBefore = usedHeap(); + long start = System.nanoTime(); + + try (Connection conn = DriverManager.getConnection(connUrl); + PreparedStatement ps = conn.prepareStatement( + "INSERT INTO " + TABLE_NAME + " (id,leid,nm,rnd) VALUES (?,?,?,?) " + + "ON DUPLICATE KEY UPDATE leid=VALUES(leid), nm=VALUES(nm), rnd=VALUES(rnd)")) { + + for (int i = 0; i < ROW_COUNT; i++) { + ps.setInt(1, i); + ps.setString(2, "LEID-" + i); + ps.setString(3, "Name-" + i); + ps.setLong(4, RND.nextLong()); + ps.executeUpdate(); + if ((i + 1) % 10_000 == 0) + System.out.printf(" %,d rows%n", i + 1); + } + } + + long elapsed = System.nanoTime() - start; + long heapDelta = usedHeap() - heapBefore; + long dbRowsAfter = countMariaDBRows(); + long fileBytes = mariaDBTableBytes(); + System.out.printf(" Elapsed: %,.0f ms | Table size: %.1f MB (InnoDB compressed+versioned) | DB rows: %,d→%,d%n", + elapsed / 1e6, fileBytes / (1024.0 * 1024.0), dbRowsBefore, dbRowsAfter); + + return new BenchResult("MariaDB versioned prepared", ROW_COUNT, elapsed, fileBytes, fileBytes, fileBytes, heapDelta, dbRowsBefore, dbRowsAfter); + } + + static BenchResult benchMariaDBVersionedBatch() throws Exception { + System.out.println("--- MariaDB versioned JDBC PreparedStatement batch ---"); + setupMariaDBVersioned(); + + String connUrl = MARIADB_URL + "&user=" + MARIADB_USER + "&password=" + MARIADB_PASS; + long dbRowsBefore = countMariaDBRows(); + long heapBefore = usedHeap(); + long start = System.nanoTime(); + + try (Connection conn = DriverManager.getConnection(connUrl); + PreparedStatement ps = conn.prepareStatement( + "INSERT INTO " + TABLE_NAME + " (id,leid,nm,rnd) VALUES (?,?,?,?) " + + "ON DUPLICATE KEY UPDATE leid=VALUES(leid), nm=VALUES(nm), rnd=VALUES(rnd)")) { + + for (int i = 0; i < ROW_COUNT; i++) { + ps.setInt(1, i); + ps.setString(2, "LEID-" + i); + ps.setString(3, "Name-" + i); + ps.setLong(4, RND.nextLong()); + ps.addBatch(); + if ((i + 1) % BATCH_SIZE == 0) { + ps.executeBatch(); + if ((i + 1) % 10_000 == 0) + System.out.printf(" %,d rows%n", i + 1); + } + } + ps.executeBatch(); + } + + long elapsed = System.nanoTime() - start; + long heapDelta = usedHeap() - heapBefore; + long dbRowsAfter = countMariaDBRows(); + long fileBytes = mariaDBTableBytes(); + System.out.printf(" Elapsed: %,.0f ms | Table size: %.1f MB (InnoDB compressed+versioned) | DB rows: %,d→%,d%n", + elapsed / 1e6, fileBytes / (1024.0 * 1024.0), dbRowsBefore, dbRowsAfter); + + return new BenchResult("MariaDB versioned batch", ROW_COUNT, elapsed, fileBytes, fileBytes, fileBytes, heapDelta, dbRowsBefore, dbRowsAfter); + } + + static BenchResult benchMariaDBVersionedLoadFile() throws Exception { + System.out.println("--- MariaDB versioned LOAD DATA INFILE (CSV) ---"); + setupMariaDBVersioned(); + + File csvFile = File.createTempFile("convex_bench_ver_", ".csv"); + csvFile.deleteOnExit(); + long writeStart = System.nanoTime(); + try (BufferedWriter bw = new BufferedWriter(new FileWriter(csvFile))) { + for (int i = 0; i < ROW_COUNT; i++) { + bw.write(i + "\tLEID-" + i + "\tName-" + i + "\t" + RND.nextLong() + "\n"); + } + } + long writeNs = System.nanoTime() - writeStart; + System.out.printf(" CSV written: %.1f MB, %,.0f ms%n", + csvFile.length() / (1024.0 * 1024.0), writeNs / 1e6); + + String connUrl = MARIADB_URL + "&user=" + MARIADB_USER + "&password=" + MARIADB_PASS + + "&allowLocalInfile=true"; + long dbRowsBefore = countMariaDBRows(); + long heapBefore = usedHeap(); + long loadStart = System.nanoTime(); + + try (Connection conn = DriverManager.getConnection(connUrl); + Statement stmt = conn.createStatement()) { + stmt.execute( + "LOAD DATA LOCAL INFILE '" + csvFile.getAbsolutePath().replace("\\", "/") + "' " + + "INTO TABLE " + TABLE_NAME + " " + + "FIELDS TERMINATED BY '\\t' " + + "LINES TERMINATED BY '\\n' " + + "(id, leid, nm, rnd)"); + } + + long loadNs = System.nanoTime() - loadStart; + long totalNs = writeNs + loadNs; + long heapDelta = usedHeap() - heapBefore; + long dbRowsAfter = countMariaDBRows(); + long fileBytes = mariaDBTableBytes(); + System.out.printf(" Load: %,.0f ms | Total: %,.0f ms | Table: %.1f MB | DB rows: %,d→%,d%n", + loadNs / 1e6, totalNs / 1e6, fileBytes / (1024.0 * 1024.0), dbRowsBefore, dbRowsAfter); + + csvFile.delete(); + return new BenchResult("MariaDB versioned LOAD FILE", ROW_COUNT, totalNs, fileBytes, fileBytes, fileBytes, heapDelta, dbRowsBefore, dbRowsAfter); + } + + // ── Single-row latency benchmarks ───────────────────────────────────────── + + static BenchResult benchConvexDirectSingle() throws Exception { + System.out.println("--- Single-row: Convex direct (plain) ---"); + // Plain SQLSchema — no history tracking, matches JDBC benchmark for fair comparison + SQLSchema schema = SQLSchema.create(); + createConvexTable(schema); + + // Warmup + for (int i = 0; i < SINGLE_WARMUP; i++) + schema.insert(TABLE_NAME, Vectors.of(CVMLong.create(i), "LEID-" + i, "Name-" + i, CVMLong.create(RND.nextLong()))); + + long start = System.nanoTime(); + for (int i = SINGLE_WARMUP; i < SINGLE_WARMUP + SINGLE_SAMPLES; i++) + schema.insert(TABLE_NAME, Vectors.of(CVMLong.create(i), "LEID-" + i, "Name-" + i, CVMLong.create(RND.nextLong()))); + long elapsed = System.nanoTime() - start; + + System.out.printf(" Avg latency: %.1f µs%n", elapsed / 1000.0 / SINGLE_SAMPLES); + return new BenchResult("Convex direct (1-row)", SINGLE_SAMPLES, elapsed, -1, -1, -1, 0); + } + + static BenchResult benchConvexPreparedSingle() throws Exception { + System.out.println("--- Single-row: Convex JDBC PreparedStatement ---"); + File storeDir = new File(STORE_DIR, "cmp-single-convex"); + SegmentedEtchStore store = SegmentedEtchStore.createFresh(storeDir); + NodeServer server = ConvexDB.createNodeServer(store); + server.launch(); + ConvexDB cdb = ConvexDB.connect(server.getCursor()); + SQLDatabase db = cdb.database(DB_NAME); + SQLSchema schema = db.tables(); + createConvexTable(schema); + cdb.register(DB_NAME); + + try (Connection conn = DriverManager.getConnection("jdbc:convex:database=" + DB_NAME); + PreparedStatement ps = conn.prepareStatement("INSERT INTO " + TABLE_NAME + " VALUES (?, ?, ?, ?)")) { + + // Warmup + for (int i = 0; i < SINGLE_WARMUP; i++) { + ps.setInt(1, i); ps.setString(2, "LEID-" + i); ps.setString(3, "Name-" + i); ps.setLong(4, RND.nextLong()); + ps.executeUpdate(); + } + + long start = System.nanoTime(); + for (int i = SINGLE_WARMUP; i < SINGLE_WARMUP + SINGLE_SAMPLES; i++) { + ps.setInt(1, i); ps.setString(2, "LEID-" + i); ps.setString(3, "Name-" + i); ps.setLong(4, RND.nextLong()); + ps.executeUpdate(); + } + long elapsed = System.nanoTime() - start; + + System.out.printf(" Avg latency: %.1f µs%n", elapsed / 1000.0 / SINGLE_SAMPLES); + cdb.unregister(DB_NAME); + server.close(); + store.close(); + return new BenchResult("Convex JDBC (1-row)", SINGLE_SAMPLES, elapsed, -1, -1, -1, 0); + } + } + + static BenchResult benchMariaDBPreparedSingle() throws Exception { + System.out.println("--- Single-row: MariaDB JDBC PreparedStatement ---"); + setupMariaDB(); + String connUrl = MARIADB_URL + "&user=" + MARIADB_USER + "&password=" + MARIADB_PASS; + + try (Connection conn = DriverManager.getConnection(connUrl); + PreparedStatement ps = conn.prepareStatement( + "INSERT INTO " + TABLE_NAME + " (id,leid,nm,rnd) VALUES (?,?,?,?) " + + "ON DUPLICATE KEY UPDATE leid=VALUES(leid), nm=VALUES(nm), rnd=VALUES(rnd)")) { + + // Warmup + for (int i = 0; i < SINGLE_WARMUP; i++) { + ps.setInt(1, i); ps.setString(2, "LEID-" + i); ps.setString(3, "Name-" + i); ps.setLong(4, RND.nextLong()); + ps.executeUpdate(); + } + + long start = System.nanoTime(); + for (int i = SINGLE_WARMUP; i < SINGLE_WARMUP + SINGLE_SAMPLES; i++) { + ps.setInt(1, i); ps.setString(2, "LEID-" + i); ps.setString(3, "Name-" + i); ps.setLong(4, RND.nextLong()); + ps.executeUpdate(); + } + long elapsed = System.nanoTime() - start; + + System.out.printf(" Avg latency: %.1f µs%n", elapsed / 1000.0 / SINGLE_SAMPLES); + return new BenchResult("MariaDB JDBC (1-row)", SINGLE_SAMPLES, elapsed, -1, -1, -1, 0); + } + } + + static BenchResult benchMariaDBLoadFileSingle() throws Exception { + System.out.println("--- Single-row: MariaDB LOAD DATA INFILE ---"); + setupMariaDB(); + String connUrl = MARIADB_URL + "&user=" + MARIADB_USER + "&password=" + MARIADB_PASS + + "&allowLocalInfile=true"; + File csvFile = File.createTempFile("convex_single_", ".csv"); + csvFile.deleteOnExit(); + + // Warmup + for (int i = 0; i < SINGLE_WARMUP; i++) { + try (BufferedWriter bw = new BufferedWriter(new FileWriter(csvFile))) { + bw.write(i + "\tLEID-" + i + "\tName-" + i + "\t" + RND.nextLong() + "\n"); + } + try (Connection conn = DriverManager.getConnection(connUrl); Statement stmt = conn.createStatement()) { + stmt.execute("LOAD DATA LOCAL INFILE '" + csvFile.getAbsolutePath().replace("\\", "/") + "' " + + "REPLACE INTO TABLE " + TABLE_NAME + " FIELDS TERMINATED BY '\\t' LINES TERMINATED BY '\\n' (id,leid,nm,rnd)"); + } + } + + long start = System.nanoTime(); + for (int i = SINGLE_WARMUP; i < SINGLE_WARMUP + SINGLE_SAMPLES; i++) { + try (BufferedWriter bw = new BufferedWriter(new FileWriter(csvFile))) { + bw.write(i + "\tLEID-" + i + "\tName-" + i + "\t" + RND.nextLong() + "\n"); + } + try (Connection conn = DriverManager.getConnection(connUrl); Statement stmt = conn.createStatement()) { + stmt.execute("LOAD DATA LOCAL INFILE '" + csvFile.getAbsolutePath().replace("\\", "/") + "' " + + "REPLACE INTO TABLE " + TABLE_NAME + " FIELDS TERMINATED BY '\\t' LINES TERMINATED BY '\\n' (id,leid,nm,rnd)"); + } + } + long elapsed = System.nanoTime() - start; + + System.out.printf(" Avg latency: %.1f µs%n", elapsed / 1000.0 / SINGLE_SAMPLES); + csvFile.delete(); + return new BenchResult("MariaDB LOAD FILE (1-row)", SINGLE_SAMPLES, elapsed, -1, -1, -1, 0); + } + + static BenchResult benchMariaDBVersionedPreparedSingle() throws Exception { + System.out.println("--- Single-row: MariaDB versioned JDBC PreparedStatement ---"); + setupMariaDBVersioned(); + String connUrl = MARIADB_URL + "&user=" + MARIADB_USER + "&password=" + MARIADB_PASS; + + try (Connection conn = DriverManager.getConnection(connUrl); + PreparedStatement ps = conn.prepareStatement( + "INSERT INTO " + TABLE_NAME + " (id,leid,nm,rnd) VALUES (?,?,?,?) " + + "ON DUPLICATE KEY UPDATE leid=VALUES(leid), nm=VALUES(nm), rnd=VALUES(rnd)")) { + + // Warmup + for (int i = 0; i < SINGLE_WARMUP; i++) { + ps.setInt(1, i); ps.setString(2, "LEID-" + i); ps.setString(3, "Name-" + i); ps.setLong(4, RND.nextLong()); + ps.executeUpdate(); + } + + long start = System.nanoTime(); + for (int i = SINGLE_WARMUP; i < SINGLE_WARMUP + SINGLE_SAMPLES; i++) { + ps.setInt(1, i); ps.setString(2, "LEID-" + i); ps.setString(3, "Name-" + i); ps.setLong(4, RND.nextLong()); + ps.executeUpdate(); + } + long elapsed = System.nanoTime() - start; + + System.out.printf(" Avg latency: %.1f µs%n", elapsed / 1000.0 / SINGLE_SAMPLES); + return new BenchResult("MariaDB versioned (1-row)", SINGLE_SAMPLES, elapsed, -1, -1, -1, 0); + } + } + + // ── Read / Update benchmarks ────────────────────────────────────────────── + + /** Convex direct: random point lookups then random updates, no JDBC overhead. */ + static List benchConvexDirectRW() throws Exception { + System.out.println("--- Read/Update: Convex direct (plain) ---"); + SQLSchema schema = SQLSchema.create(); // plain — matches JDBC RW benchmarks + createConvexTable(schema); + + // Load dataset + for (int i = 0; i < ROW_COUNT; i++) + schema.insert(TABLE_NAME, Vectors.of(CVMLong.create(i), "LEID-" + i, "Name-" + i, CVMLong.create(RND.nextLong()))); + System.out.printf(" Loaded %,d rows%n", ROW_COUNT); + + // Lookups + long t0 = System.nanoTime(); + for (int i = 0; i < RW_OPS; i++) { + long id = Math.abs(RND.nextLong()) % ROW_COUNT; + schema.selectByKey(TABLE_NAME, CVMLong.create(id)); + } + long lookupNs = System.nanoTime() - t0; + System.out.printf(" Lookup avg: %.1f µs%n", lookupNs / 1000.0 / RW_OPS); + + // Updates + long t1 = System.nanoTime(); + for (int i = 0; i < RW_OPS; i++) { + long id = Math.abs(RND.nextLong()) % ROW_COUNT; + schema.insert(TABLE_NAME, Vectors.of(CVMLong.create(id), "LEID-" + id, "Name-" + id, CVMLong.create(RND.nextLong()))); + } + long updateNs = System.nanoTime() - t1; + System.out.printf(" Update avg: %.1f µs%n", updateNs / 1000.0 / RW_OPS); + + return List.of( + new BenchResult("Convex direct lookup", RW_OPS, lookupNs, -1, -1, -1, 0), + new BenchResult("Convex direct update", RW_OPS, updateNs, -1, -1, -1, 0) + ); + } + + /** Convex JDBC: random SELECT by PK and UPDATE via PreparedStatement. */ + static List benchConvexJdbcRW() throws Exception { + System.out.println("--- Read/Update: Convex JDBC ---"); + File storeDir = new File(STORE_DIR, "cmp-rw-convex"); + SegmentedEtchStore store = SegmentedEtchStore.createFresh(storeDir); + NodeServer server = ConvexDB.createNodeServer(store); + server.launch(); + ConvexDB cdb = ConvexDB.connect(server.getCursor()); + SQLDatabase db = cdb.database(DB_NAME); + SQLSchema schema = db.tables(); + createConvexTable(schema); + cdb.register(DB_NAME); + + // Load dataset via lattice (faster setup) + for (int i = 0; i < ROW_COUNT; i++) + schema.insert(TABLE_NAME, Vectors.of(CVMLong.create(i), "LEID-" + i, "Name-" + i, CVMLong.create(RND.nextLong()))); + System.out.printf(" Loaded %,d rows%n", ROW_COUNT); + + try (Connection conn = DriverManager.getConnection("jdbc:convex:database=" + DB_NAME); + PreparedStatement sel = conn.prepareStatement("SELECT ID, LEID, NM, RND FROM " + TABLE_NAME + " WHERE ID = ?"); + PreparedStatement upd = conn.prepareStatement("INSERT INTO " + TABLE_NAME + " VALUES (?, ?, ?, ?)")) { + + // Lookups + long t0 = System.nanoTime(); + for (int i = 0; i < RW_OPS; i++) { + sel.setLong(1, Math.abs(RND.nextLong()) % ROW_COUNT); + sel.executeQuery().close(); + } + long lookupNs = System.nanoTime() - t0; + System.out.printf(" Lookup avg: %.1f µs%n", lookupNs / 1000.0 / RW_OPS); + + // Updates + long t1 = System.nanoTime(); + for (int i = 0; i < RW_OPS; i++) { + long id = Math.abs(RND.nextLong()) % ROW_COUNT; + upd.setLong(1, id); upd.setString(2, "LEID-" + id); upd.setString(3, "Name-" + id); upd.setLong(4, RND.nextLong()); + upd.executeUpdate(); + } + long updateNs = System.nanoTime() - t1; + System.out.printf(" Update avg: %.1f µs%n", updateNs / 1000.0 / RW_OPS); + + cdb.unregister(DB_NAME); + server.close(); + store.close(); + return List.of( + new BenchResult("Convex JDBC lookup", RW_OPS, lookupNs, -1, -1, -1, 0), + new BenchResult("Convex JDBC update", RW_OPS, updateNs, -1, -1, -1, 0) + ); + } + } + + /** MariaDB: random SELECT by PK and UPDATE via PreparedStatement. */ + static List benchMariaDBRW(boolean versioned) throws Exception { + String tag = versioned ? "versioned" : "plain"; + System.out.println("--- Read/Update: MariaDB " + tag + " ---"); + if (versioned) setupMariaDBVersioned(); else setupMariaDB(); + + // Load dataset via batch + String connUrl = MARIADB_URL + "&user=" + MARIADB_USER + "&password=" + MARIADB_PASS; + try (Connection conn = DriverManager.getConnection(connUrl); + PreparedStatement ins = conn.prepareStatement( + "INSERT INTO " + TABLE_NAME + " (id,leid,nm,rnd) VALUES (?,?,?,?) " + + "ON DUPLICATE KEY UPDATE leid=VALUES(leid), nm=VALUES(nm), rnd=VALUES(rnd)")) { + for (int i = 0; i < ROW_COUNT; i++) { + ins.setInt(1, i); ins.setString(2, "LEID-" + i); ins.setString(3, "Name-" + i); ins.setLong(4, RND.nextLong()); + ins.addBatch(); + if ((i + 1) % BATCH_SIZE == 0) ins.executeBatch(); + } + ins.executeBatch(); + } + System.out.printf(" Loaded %,d rows%n", ROW_COUNT); + + try (Connection conn = DriverManager.getConnection(connUrl); + PreparedStatement sel = conn.prepareStatement("SELECT id, leid, nm, rnd FROM " + TABLE_NAME + " WHERE id = ?"); + PreparedStatement upd = conn.prepareStatement( + "UPDATE " + TABLE_NAME + " SET leid=?, nm=?, rnd=? WHERE id=?")) { + + // Lookups + long t0 = System.nanoTime(); + for (int i = 0; i < RW_OPS; i++) { + sel.setLong(1, Math.abs(RND.nextLong()) % ROW_COUNT); + sel.executeQuery().close(); + } + long lookupNs = System.nanoTime() - t0; + System.out.printf(" Lookup avg: %.1f µs%n", lookupNs / 1000.0 / RW_OPS); + + // Updates + long t1 = System.nanoTime(); + for (int i = 0; i < RW_OPS; i++) { + long id = Math.abs(RND.nextLong()) % ROW_COUNT; + upd.setString(1, "LEID-" + id); upd.setString(2, "Name-" + id); upd.setLong(3, RND.nextLong()); upd.setLong(4, id); + upd.executeUpdate(); + } + long updateNs = System.nanoTime() - t1; + System.out.printf(" Update avg: %.1f µs%n", updateNs / 1000.0 / RW_OPS); + + return List.of( + new BenchResult("MariaDB " + tag + " lookup", RW_OPS, lookupNs, -1, -1, -1, 0), + new BenchResult("MariaDB " + tag + " update", RW_OPS, updateNs, -1, -1, -1, 0) + ); + } + } + + /** + * Full table scan benchmark: measures heap consumed while iterating SELECT * on ROW_COUNT rows. + * The heap spike between executeQuery() and first rs.next() reveals pre-collection overhead. + * With the streaming enumerator this should be near zero; with the old list-based one it is O(n). + */ + static BenchResult benchConvexJdbcScan() throws Exception { + System.out.println("--- Full scan: Convex JDBC SELECT * ---"); + File storeDir = new File(STORE_DIR, "cmp-scan-convex"); + SegmentedEtchStore store = SegmentedEtchStore.createFresh(storeDir); + NodeServer server = ConvexDB.createNodeServer(store); + server.launch(); + ConvexDB cdb = ConvexDB.connect(server.getCursor()); + SQLDatabase db = cdb.database(DB_NAME); + SQLSchema schema = db.tables(); + createConvexTable(schema); + cdb.register(DB_NAME); + + // Use batch insert so blockVec is built inline and persisted to Etch. + // After gc() the scan can load the compact VectorTree instead of the full Index trie. + List> scanBatch = new ArrayList<>(ROW_COUNT); + for (int i = 0; i < ROW_COUNT; i++) + scanBatch.add(Vectors.of(CVMLong.create(i), "LEID-" + i, "Name-" + i, CVMLong.create(RND.nextLong()))); + schema.insertAll(TABLE_NAME, scanBatch); + scanBatch = null; // allow GC of input list + System.out.printf(" Loaded %,d rows%n", ROW_COUNT); + + System.gc(); Thread.sleep(100); System.gc(); + long heapBefore = usedHeap(); + long start = System.nanoTime(); + long heapPeak = heapBefore; + int count = 0; + + try (Connection conn = DriverManager.getConnection("jdbc:convex:database=" + DB_NAME); + Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("SELECT * FROM " + TABLE_NAME); + // Track peak heap during full scan (lazy scan: rows decoded on rs.next()) + while (rs.next()) { + count++; + if ((count & 0xFFF) == 0) heapPeak = Math.max(heapPeak, usedHeap()); + } + heapPeak = Math.max(heapPeak, usedHeap()); + rs.close(); + } + + long elapsed = System.nanoTime() - start; + long heapDelta = heapPeak - heapBefore; + System.out.printf(" Rows: %,d | Elapsed: %,.0f ms | Peak heap: +%.1f MB%n", + count, elapsed / 1e6, heapDelta / (1024.0 * 1024.0)); + + cdb.unregister(DB_NAME); + server.close(); + store.close(); + return new BenchResult("Convex JDBC scan", ROW_COUNT, elapsed, -1, -1, -1, heapDelta); + } + + // ── Merge latency benchmark ─────────────────────────────────────────────── + + /** + * Times in-JVM lattice merge: full (empty→ROW_COUNT) then incremental (k changes). + * Verifies convergence after each merge (A's value == B's value). + */ + @SuppressWarnings({"unchecked", "rawtypes"}) + static List benchConvexMergeLatency() throws Exception { + System.out.println("--- Convex merge latency (in-JVM: full + incremental) ---"); + List results = new ArrayList<>(); + + // ── Build base on nodeA ────────────────────────────────────────── + File dirA = new File(STORE_DIR, "cmp-merge-nodeA"); + SegmentedEtchStore storeA = SegmentedEtchStore.createFresh(dirA); + NodeServer serverA = ConvexDB.createNodeServer(storeA); + serverA.launch(); + SQLDatabase dbA = SQLDatabase.connect(serverA.getCursor(), DB_NAME); + SQLSchema schemaA = dbA.tables(); + createConvexTable(schemaA); + + List> batch = new ArrayList<>(ROW_COUNT); + for (int i = 0; i < ROW_COUNT; i++) + batch.add(Vectors.of(CVMLong.create(i), "LEID-" + i, "Name-" + i, CVMLong.create(RND.nextLong()))); + schemaA.insertAll(TABLE_NAME, batch); + System.out.printf(" NodeA loaded: %,d rows%n", ROW_COUNT); + + // ── NodeB: empty ───────────────────────────────────────────────── + File dirB = new File(STORE_DIR, "cmp-merge-nodeB"); + SegmentedEtchStore storeB = SegmentedEtchStore.createFresh(dirB); + NodeServer serverB = ConvexDB.createNodeServer(storeB); + serverB.launch(); + + // ── Full merge: A (ROW_COUNT rows) → B (empty) ─────────────────── + System.gc(); Thread.sleep(50); + long fullStart = System.nanoTime(); + serverB.mergeValue(serverA.getLocalValue()); + long fullNs = System.nanoTime() - fullStart; + + boolean fullConverged = serverA.getLocalValue().equals(serverB.getLocalValue()); + System.out.printf(" Full merge (%,d rows): %,.0f ms converged=%b%n", + ROW_COUNT, fullNs / 1e6, fullConverged); + results.add(new BenchResult("Convex merge full", ROW_COUNT, fullNs, -1, -1, -1, 0)); + + // ── Incremental merges: apply k changes to A, merge to B ───────── + for (int k : REPLICATION_K) { + for (int i = 0; i < k; i++) { + long id = Math.abs(RND.nextLong()) % ROW_COUNT; + schemaA.insert(TABLE_NAME, Vectors.of(CVMLong.create(id), + "UPD-" + id, "Name-" + id, CVMLong.create(RND.nextLong()))); + } + System.gc(); Thread.sleep(50); + long incrStart = System.nanoTime(); + serverB.mergeValue(serverA.getLocalValue()); + long incrNs = System.nanoTime() - incrStart; + + boolean incrConverged = serverA.getLocalValue().equals(serverB.getLocalValue()); + System.out.printf(" Incremental k=%,6d: %,.0f ms converged=%b%n", + k, incrNs / 1e6, incrConverged); + results.add(new BenchResult("Convex merge k=" + k, k, incrNs, -1, -1, -1, 0)); + } + + serverA.close(); storeA.close(); + serverB.close(); storeB.close(); + return results; + } + + static BenchResult benchMariaDBScan() throws Exception { + System.out.println("--- Full scan: MariaDB SELECT * ---"); + setupMariaDB(); + String connUrl = MARIADB_URL + "&user=" + MARIADB_USER + "&password=" + MARIADB_PASS; + try (Connection conn = DriverManager.getConnection(connUrl); + PreparedStatement ins = conn.prepareStatement( + "INSERT INTO " + TABLE_NAME + " (id,leid,nm,rnd) VALUES (?,?,?,?) " + + "ON DUPLICATE KEY UPDATE leid=VALUES(leid), nm=VALUES(nm), rnd=VALUES(rnd)")) { + for (int i = 0; i < ROW_COUNT; i++) { + ins.setInt(1, i); ins.setString(2, "LEID-" + i); ins.setString(3, "Name-" + i); ins.setLong(4, RND.nextLong()); + ins.addBatch(); + if ((i + 1) % BATCH_SIZE == 0) ins.executeBatch(); + } + ins.executeBatch(); + } + System.out.printf(" Loaded %,d rows%n", ROW_COUNT); + + System.gc(); Thread.sleep(100); System.gc(); + long heapBefore = usedHeap(); + long start = System.nanoTime(); + long heapPeak = heapBefore; + int count = 0; + + try (Connection conn = DriverManager.getConnection(connUrl); + Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("SELECT * FROM " + TABLE_NAME); + while (rs.next()) { + count++; + if ((count & 0xFFF) == 0) heapPeak = Math.max(heapPeak, usedHeap()); + } + heapPeak = Math.max(heapPeak, usedHeap()); + rs.close(); + } + + long elapsed = System.nanoTime() - start; + long heapDelta = heapPeak - heapBefore; + System.out.printf(" Rows: %,d | Elapsed: %,.0f ms | Peak heap: +%.1f MB%n", + count, elapsed / 1e6, heapDelta / (1024.0 * 1024.0)); + return new BenchResult("MariaDB scan", ROW_COUNT, elapsed, -1, -1, -1, heapDelta); + } + + // ── MariaDB MEMORY engine benchmarks ───────────────────────────────────── + + static void setupMariaDBMemory() throws Exception { + String connUrl = MARIADB_URL + "&user=" + MARIADB_USER + "&password=" + MARIADB_PASS; + try (Connection conn = DriverManager.getConnection(connUrl); + Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP TABLE IF EXISTS " + TABLE_NAME_MEM); + stmt.executeUpdate( + "CREATE TABLE " + TABLE_NAME_MEM + " (" + + "id BIGINT PRIMARY KEY, " + + "leid VARCHAR(32), " + + "nm VARCHAR(32), " + + "rnd BIGINT" + + ") ENGINE=MEMORY"); + } + } + + static long countMariaDBMemoryRows() { + String connUrl = MARIADB_URL + "&user=" + MARIADB_USER + "&password=" + MARIADB_PASS; + try (Connection conn = DriverManager.getConnection(connUrl); + Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("SELECT COUNT(*) FROM " + TABLE_NAME_MEM); + if (rs.next()) return rs.getLong(1); + } catch (Exception e) { /* table may not exist */ } + return -1; + } + + static BenchResult benchMariaDBMemoryPrepared() throws Exception { + System.out.println("--- MariaDB MEMORY JDBC PreparedStatement ---"); + setupMariaDBMemory(); + String connUrl = MARIADB_URL + "&user=" + MARIADB_USER + "&password=" + MARIADB_PASS; + long rowsBefore = countMariaDBMemoryRows(); + long heapBefore = usedHeap(); + long start = System.nanoTime(); + + try (Connection conn = DriverManager.getConnection(connUrl); + PreparedStatement ps = conn.prepareStatement( + "INSERT INTO " + TABLE_NAME_MEM + " (id,leid,nm,rnd) VALUES (?,?,?,?) " + + "ON DUPLICATE KEY UPDATE leid=VALUES(leid), nm=VALUES(nm), rnd=VALUES(rnd)")) { + for (int i = 0; i < ROW_COUNT; i++) { + ps.setInt(1, i); ps.setString(2, "LEID-" + i); + ps.setString(3, "Name-" + i); ps.setLong(4, RND.nextLong()); + ps.executeUpdate(); + if ((i + 1) % 10_000 == 0) System.out.printf(" %,d rows%n", i + 1); + } + } + + long elapsed = System.nanoTime() - start; + long heapDelta = usedHeap() - heapBefore; + long rowsAfter = countMariaDBMemoryRows(); + System.out.printf(" Elapsed: %,.0f ms | Engine: MEMORY (no disk) | DB rows: %,d→%,d%n", + elapsed / 1e6, rowsBefore, rowsAfter); + return new BenchResult("MariaDB MEMORY prepared", ROW_COUNT, elapsed, -1, -1, -1, heapDelta, rowsBefore, rowsAfter); + } + + static BenchResult benchMariaDBMemoryBatch() throws Exception { + System.out.println("--- MariaDB MEMORY JDBC PreparedStatement batch ---"); + setupMariaDBMemory(); + String connUrl = MARIADB_URL + "&user=" + MARIADB_USER + "&password=" + MARIADB_PASS; + long rowsBefore = countMariaDBMemoryRows(); + long heapBefore = usedHeap(); + long start = System.nanoTime(); + + try (Connection conn = DriverManager.getConnection(connUrl); + PreparedStatement ps = conn.prepareStatement( + "INSERT INTO " + TABLE_NAME_MEM + " (id,leid,nm,rnd) VALUES (?,?,?,?) " + + "ON DUPLICATE KEY UPDATE leid=VALUES(leid), nm=VALUES(nm), rnd=VALUES(rnd)")) { + for (int i = 0; i < ROW_COUNT; i++) { + ps.setInt(1, i); ps.setString(2, "LEID-" + i); + ps.setString(3, "Name-" + i); ps.setLong(4, RND.nextLong()); + ps.addBatch(); + if ((i + 1) % BATCH_SIZE == 0) { + ps.executeBatch(); + if ((i + 1) % 10_000 == 0) System.out.printf(" %,d rows%n", i + 1); + } + } + ps.executeBatch(); + } + + long elapsed = System.nanoTime() - start; + long heapDelta = usedHeap() - heapBefore; + long rowsAfter = countMariaDBMemoryRows(); + System.out.printf(" Elapsed: %,.0f ms | Engine: MEMORY (no disk) | DB rows: %,d→%,d%n", + elapsed / 1e6, rowsBefore, rowsAfter); + return new BenchResult("MariaDB MEMORY batch", ROW_COUNT, elapsed, -1, -1, -1, heapDelta, rowsBefore, rowsAfter); + } + + static BenchResult benchMariaDBMemoryPreparedSingle() throws Exception { + System.out.println("--- Single-row latency: MariaDB MEMORY ---"); + setupMariaDBMemory(); + String connUrl = MARIADB_URL + "&user=" + MARIADB_USER + "&password=" + MARIADB_PASS; + + try (Connection conn = DriverManager.getConnection(connUrl); + PreparedStatement ps = conn.prepareStatement( + "INSERT INTO " + TABLE_NAME_MEM + " (id,leid,nm,rnd) VALUES (?,?,?,?) " + + "ON DUPLICATE KEY UPDATE leid=VALUES(leid), nm=VALUES(nm), rnd=VALUES(rnd)")) { + // Warmup + for (int i = 0; i < SINGLE_WARMUP; i++) { + ps.setInt(1, i); ps.setString(2, "LEID-" + i); + ps.setString(3, "Name-" + i); ps.setLong(4, RND.nextLong()); + ps.executeUpdate(); + } + // Timed + long start = System.nanoTime(); + for (int i = 0; i < SINGLE_SAMPLES; i++) { + int id = SINGLE_WARMUP + i; + ps.setInt(1, id); ps.setString(2, "LEID-" + id); + ps.setString(3, "Name-" + id); ps.setLong(4, RND.nextLong()); + ps.executeUpdate(); + } + long elapsed = System.nanoTime() - start; + double usPerRow = elapsed / 1000.0 / SINGLE_SAMPLES; + System.out.printf(" Avg: %.1f µs/row (%,.0f rows/sec)%n", + usPerRow, 1_000_000.0 / usPerRow); + return new BenchResult("MariaDB MEMORY single", SINGLE_SAMPLES, elapsed, -1, -1, -1, 0); + } + } + + static List benchMariaDBMemoryRW() throws Exception { + System.out.println("--- Read/Update: MariaDB MEMORY ---"); + setupMariaDBMemory(); + String connUrl = MARIADB_URL + "&user=" + MARIADB_USER + "&password=" + MARIADB_PASS; + + try (Connection conn = DriverManager.getConnection(connUrl); + PreparedStatement ins = conn.prepareStatement( + "INSERT INTO " + TABLE_NAME_MEM + " (id,leid,nm,rnd) VALUES (?,?,?,?) " + + "ON DUPLICATE KEY UPDATE leid=VALUES(leid), nm=VALUES(nm), rnd=VALUES(rnd)")) { + for (int i = 0; i < ROW_COUNT; i++) { + ins.setInt(1, i); ins.setString(2, "LEID-" + i); + ins.setString(3, "Name-" + i); ins.setLong(4, RND.nextLong()); + ins.addBatch(); + if ((i + 1) % BATCH_SIZE == 0) ins.executeBatch(); + } + ins.executeBatch(); + } + System.out.printf(" Loaded %,d rows%n", ROW_COUNT); + + try (Connection conn = DriverManager.getConnection(connUrl); + PreparedStatement sel = conn.prepareStatement( + "SELECT id,leid,nm,rnd FROM " + TABLE_NAME_MEM + " WHERE id=?"); + PreparedStatement upd = conn.prepareStatement( + "UPDATE " + TABLE_NAME_MEM + " SET leid=?,nm=?,rnd=? WHERE id=?")) { + + long t0 = System.nanoTime(); + for (int i = 0; i < RW_OPS; i++) { + sel.setLong(1, Math.abs(RND.nextLong()) % ROW_COUNT); + sel.executeQuery().close(); + } + long lookupNs = System.nanoTime() - t0; + System.out.printf(" Lookup avg: %.1f µs%n", lookupNs / 1000.0 / RW_OPS); + + long t1 = System.nanoTime(); + for (int i = 0; i < RW_OPS; i++) { + long id = Math.abs(RND.nextLong()) % ROW_COUNT; + upd.setString(1, "LEID-" + id); upd.setString(2, "Name-" + id); + upd.setLong(3, RND.nextLong()); upd.setLong(4, id); + upd.executeUpdate(); + } + long updateNs = System.nanoTime() - t1; + System.out.printf(" Update avg: %.1f µs%n", updateNs / 1000.0 / RW_OPS); + + return List.of( + new BenchResult("MariaDB MEMORY lookup", RW_OPS, lookupNs, -1, -1, -1, 0), + new BenchResult("MariaDB MEMORY update", RW_OPS, updateNs, -1, -1, -1, 0) + ); + } + } + + static BenchResult benchMariaDBMemoryScan() throws Exception { + System.out.println("--- Full scan: MariaDB MEMORY SELECT * ---"); + setupMariaDBMemory(); + String connUrl = MARIADB_URL + "&user=" + MARIADB_USER + "&password=" + MARIADB_PASS; + + try (Connection conn = DriverManager.getConnection(connUrl); + PreparedStatement ins = conn.prepareStatement( + "INSERT INTO " + TABLE_NAME_MEM + " (id,leid,nm,rnd) VALUES (?,?,?,?) " + + "ON DUPLICATE KEY UPDATE leid=VALUES(leid), nm=VALUES(nm), rnd=VALUES(rnd)")) { + for (int i = 0; i < ROW_COUNT; i++) { + ins.setInt(1, i); ins.setString(2, "LEID-" + i); + ins.setString(3, "Name-" + i); ins.setLong(4, RND.nextLong()); + ins.addBatch(); + if ((i + 1) % BATCH_SIZE == 0) ins.executeBatch(); + } + ins.executeBatch(); + } + System.out.printf(" Loaded %,d rows%n", ROW_COUNT); + + System.gc(); Thread.sleep(100); System.gc(); + long heapBefore = usedHeap(); + long start = System.nanoTime(); + long heapPeak = heapBefore; + int count = 0; + + try (Connection conn = DriverManager.getConnection(connUrl); + Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("SELECT * FROM " + TABLE_NAME_MEM); + while (rs.next()) { + count++; + if ((count & 0xFFF) == 0) heapPeak = Math.max(heapPeak, usedHeap()); + } + heapPeak = Math.max(heapPeak, usedHeap()); + rs.close(); + } + + long elapsed = System.nanoTime() - start; + long heapDelta = heapPeak - heapBefore; + System.out.printf(" Rows: %,d | Elapsed: %,.0f ms | Peak heap: +%.1f MB%n", + count, elapsed / 1e6, heapDelta / (1024.0 * 1024.0)); + return new BenchResult("MariaDB MEMORY scan", ROW_COUNT, elapsed, -1, -1, -1, heapDelta); + } + + static void printScanComparison(List results) { + System.out.println("\n=== Full scan results ==="); + System.out.println("╔══════════════════════════════╦══════════════╦══════════════╦══════════════╗"); + System.out.println("║ Benchmark ║ rows/sec ║ peak heap ║ heap/row ║"); + System.out.println("║ ║ ║ (MB) ║ (bytes) ║"); + System.out.println("╠══════════════════════════════╬══════════════╬══════════════╬══════════════╣"); + for (BenchResult r : results) { + double heapMB = r.heapDeltaBytes() / (1024.0 * 1024.0); + double heapPerRow = r.heapDeltaBytes() > 0 ? (double) r.heapDeltaBytes() / r.rows() : 0; + System.out.printf("║ %-28s ║ %,12.0f ║ %,12.1f ║ %,12.1f ║%n", + r.label(), r.rowsPerSec(), heapMB, heapPerRow); + } + System.out.println("╚══════════════════════════════╩══════════════╩══════════════╩══════════════╝"); + } + + static void printRWComparison(List results) { + // Print lookup then update, each split into plain/versioned sub-groups + for (String kind : new String[]{"lookup", "update"}) { + System.out.printf(" — %s —%n", kind.toUpperCase()); + System.out.println("╔══════════════════════════════╦══════════════╦══════════════╗"); + System.out.println("║ Benchmark ║ avg µs/op ║ ops/sec ║"); + System.out.println("╠══════════════════════════════╬══════════════╬══════════════╣"); + for (boolean versionedGroup : new boolean[]{false, true}) { + List group = results.stream() + .filter(r -> r.label().endsWith(kind) && isVersioned(r.label()) == versionedGroup) + .toList(); + if (group.isEmpty()) continue; + String gLabel = versionedGroup ? "VERSIONED" : "PLAIN"; + System.out.printf("║ ── %-56s ║%n", gLabel); + for (BenchResult r : group) { + double usPerOp = r.elapsedNs() / 1000.0 / r.rows(); + System.out.printf("║ %-28s ║ %,12.1f ║ %,12.0f ║%n", r.label(), usPerOp, r.rowsPerSec()); + } + } + System.out.println("╚══════════════════════════════╩══════════════╩══════════════╝"); + System.out.println(); + } + + // Ratio: Convex direct lookup vs plain MariaDB lookup + BenchResult cdl = results.stream().filter(r -> r.label().equals("Convex direct lookup")).findFirst().orElse(null); + BenchResult cdu = results.stream().filter(r -> r.label().equals("Convex direct update")).findFirst().orElse(null); + BenchResult mpl = results.stream().filter(r -> r.label().equals("MariaDB plain lookup")).findFirst().orElse(null); + BenchResult mpu = results.stream().filter(r -> r.label().equals("MariaDB plain update")).findFirst().orElse(null); + if (cdl != null && mpl != null) + System.out.printf(" Convex direct vs MariaDB plain — lookup: %.1fx update: %.1fx%n", + (double) mpl.elapsedNs() / cdl.elapsedNs(), + mpu != null && cdu != null ? (double) mpu.elapsedNs() / cdu.elapsedNs() : Double.NaN); + } + + // ── Replication / Dedup benchmark ───────────────────────────────────────── + + /** + * Measures the incremental sync payload for k changes on a full ROW_COUNT dataset. + * + *

For Convex: the payload is the additional bytes written to the Etch store after + * k updates — i.e. the new Merkle tree nodes created. Content-addressing means + * unchanged sub-trees cost zero bandwidth: O(k·log n) new cells vs O(n) for a full scan. + * + *

For MariaDB: uses the binlog position delta if binary logging is enabled; + * falls back to k × avg_row_size estimate. + * + *

For PostgreSQL: uses pg_wal_lsn_diff to measure exact WAL bytes written. + */ + static void benchmarkReplication(boolean mariaAvailable, boolean psqlAvailable) throws Exception { + System.out.println("\n=== Replication / Dedup Benchmark ==="); + System.out.printf("Incremental sync payload for k changes on %,d-row dataset%n", ROW_COUNT); + System.out.printf("k values: %s%n%n", Arrays.toString(REPLICATION_K)); + + List results = new ArrayList<>(); + + // ── Convex ──────────────────────────────────────────────────────── + File pristineFile = new File(STORE_DIR, "cmp-repl-pristine.etch"); + System.out.println(" Building Convex base (" + String.format("%,d", ROW_COUNT) + " rows)..."); + buildConvexReplBase(pristineFile); + long convexBaseDataBytes = etchDataBytes(pristineFile); + long convexFullBytes = pristineFile.length(); + System.out.printf(" Convex base: %.1f MB written, %.1f MB allocated%n", + convexBaseDataBytes / (1024.0 * 1024.0), convexFullBytes / (1024.0 * 1024.0)); + + for (int k : REPLICATION_K) { + System.out.printf(" Convex k=%,d ...", k); + File workFile = new File(STORE_DIR, "cmp-repl-work.etch"); + Files.copy(pristineFile.toPath(), workFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + + EtchStore store = EtchStore.create(workFile); + NodeServer server = ConvexDB.createNodeServer(store); + server.launch(); + SQLDatabase db = SQLDatabase.connect(server.getCursor(), DB_NAME); + SQLSchema schema = db.tables(); // plain — consistent with base + + for (int i = 0; i < k; i++) { + long id = Math.abs(RND.nextLong()) % ROW_COUNT; + schema.insert(TABLE_NAME, Vectors.of(CVMLong.create(id), + "REPL-" + id, "Updated-" + id, CVMLong.create(RND.nextLong()))); + } + server.persistSnapshot(server.getLocalValue()); + long payloadBytes = etchDataBytes(workFile) - convexBaseDataBytes; + server.close(); + store.close(); + workFile.delete(); + + System.out.printf(" payload=%.1f KB (%.0f B/change)%n", payloadBytes / 1024.0, (double) payloadBytes / k); + results.add(new ReplicationResult("Convex", k, payloadBytes, convexBaseDataBytes, + "etchDataBytes delta: new Merkle cells for " + k + " updates")); + } + + // ── MariaDB ─────────────────────────────────────────────────────── + if (mariaAvailable) { + System.out.println(" Loading MariaDB base dataset..."); + setupMariaDB(); + long mariaFullBytes = loadMariaDBDataset(); + System.out.printf(" MariaDB base: %.1f MB%n", mariaFullBytes / (1024.0 * 1024.0)); + long avgRowBytes = mariaFullBytes / ROW_COUNT; + + for (int k : REPLICATION_K) { + // Apply k updates (we measure two scenarios, not actual binlog) + applyMariaDBUpdates(k); + // Scenario A: with binlog/changelog — O(k·row) payload estimate + long withChangelogBytes = k * avgRowBytes; + // Scenario B: no changelog — replica must do full table scan = O(n·row) + long noChangelogBytes = mariaFullBytes; + System.out.printf(" MariaDB k=%,d changelog est=%.1f KB full-scan=%.1f MB%n", + k, withChangelogBytes / 1024.0, noChangelogBytes / (1024.0 * 1024.0)); + results.add(new ReplicationResult("MariaDB+changelog", k, withChangelogBytes, mariaFullBytes, + "estimate: k × avg_row_size (" + avgRowBytes + " B/row)")); + results.add(new ReplicationResult("MariaDB no-log", k, noChangelogBytes, mariaFullBytes, + "full table scan (no binlog)")); + } + } + + // ── PostgreSQL ──────────────────────────────────────────────────── + if (psqlAvailable) { + System.out.println(" Loading PostgreSQL base dataset..."); + setupPostgres(); + long pgFullBytes = loadPostgresDataset(); + System.out.printf(" PostgreSQL base: %.1f MB%n", pgFullBytes / (1024.0 * 1024.0)); + + for (int k : REPLICATION_K) { + System.out.printf(" PostgreSQL k=%,d ...", k); + long payloadBytes = measurePostgresReplicationPayload(k); + System.out.printf(" WAL payload=%.1f KB (%.0f B/change)%n", + payloadBytes / 1024.0, (double) payloadBytes / k); + results.add(new ReplicationResult("PostgreSQL", k, payloadBytes, pgFullBytes, + "pg_wal_lsn_diff: WAL bytes for " + k + " updates")); + } + } + + printReplicationComparison(results); + } + + /** Builds a pristine ROW_COUNT-row Convex Etch store for replication baseline. */ + static void buildConvexReplBase(File storeFile) throws Exception { + EtchStore store = EtchStore.create(storeFile); + NodeServer server = ConvexDB.createNodeServer(store); + server.launch(); + SQLDatabase db = SQLDatabase.connect(server.getCursor(), DB_NAME); + SQLSchema schema = db.tables(); // plain — no history, measures pure Merkle path overhead + createConvexTable(schema); + + for (int i = 0; i < ROW_COUNT; i++) + schema.insert(TABLE_NAME, Vectors.of(CVMLong.create(i), + "LEID-" + i, "Name-" + i, CVMLong.create(RND.nextLong()))); + + server.persistSnapshot(server.getLocalValue()); + server.close(); + store.close(); + } + + /** Loads ROW_COUNT rows into MariaDB (batch mode) and returns table bytes. */ + static long loadMariaDBDataset() throws Exception { + String connUrl = MARIADB_URL + "&user=" + MARIADB_USER + "&password=" + MARIADB_PASS; + try (Connection conn = DriverManager.getConnection(connUrl); + PreparedStatement ins = conn.prepareStatement( + "INSERT INTO " + TABLE_NAME + " (id,leid,nm,rnd) VALUES (?,?,?,?) " + + "ON DUPLICATE KEY UPDATE leid=VALUES(leid), nm=VALUES(nm), rnd=VALUES(rnd)")) { + for (int i = 0; i < ROW_COUNT; i++) { + ins.setInt(1, i); ins.setString(2, "LEID-" + i); ins.setString(3, "Name-" + i); + ins.setLong(4, RND.nextLong()); + ins.addBatch(); + if ((i + 1) % BATCH_SIZE == 0) ins.executeBatch(); + } + ins.executeBatch(); + } + return mariaDBTableBytes(); + } + + /** Applies k random updates to MariaDB. */ + static void applyMariaDBUpdates(int k) throws Exception { + String connUrl = MARIADB_URL + "&user=" + MARIADB_USER + "&password=" + MARIADB_PASS; + try (Connection conn = DriverManager.getConnection(connUrl); + PreparedStatement upd = conn.prepareStatement( + "UPDATE " + TABLE_NAME + " SET leid=?, nm=?, rnd=? WHERE id=?")) { + for (int i = 0; i < k; i++) { + long id = Math.abs(RND.nextLong()) % ROW_COUNT; + upd.setString(1, "REPL-" + id); upd.setString(2, "Upd-" + id); + upd.setLong(3, RND.nextLong()); upd.setLong(4, id); + upd.addBatch(); + if ((i + 1) % BATCH_SIZE == 0) upd.executeBatch(); + } + upd.executeBatch(); + } + } + + /** Loads ROW_COUNT rows into PostgreSQL (batch mode) and returns table bytes. */ + static long loadPostgresDataset() throws Exception { + try (Connection conn = DriverManager.getConnection(PSQL_URL, PSQL_USER, PSQL_PASS); + PreparedStatement ins = conn.prepareStatement( + "INSERT INTO " + TABLE_NAME + " (id,leid,nm,rnd) VALUES (?,?,?,?) " + + "ON CONFLICT (id) DO UPDATE SET leid=EXCLUDED.leid, nm=EXCLUDED.nm, rnd=EXCLUDED.rnd")) { + for (int i = 0; i < ROW_COUNT; i++) { + ins.setInt(1, i); ins.setString(2, "LEID-" + i); ins.setString(3, "Name-" + i); + ins.setLong(4, RND.nextLong()); + ins.addBatch(); + if ((i + 1) % BATCH_SIZE == 0) ins.executeBatch(); + } + ins.executeBatch(); + } + return postgresTableBytes(); + } + + /** + * Applies k random updates to PostgreSQL and returns WAL bytes written + * as measured by pg_wal_lsn_diff. + */ + static long measurePostgresReplicationPayload(int k) throws Exception { + try (Connection conn = DriverManager.getConnection(PSQL_URL, PSQL_USER, PSQL_PASS); + Statement stmt = conn.createStatement()) { + + // Capture WAL LSN before + ResultSet rs = stmt.executeQuery("SELECT pg_current_wal_lsn() AS lsn"); + rs.next(); + String beforeLsn = rs.getString("lsn"); + + // Apply k random updates + try (PreparedStatement upd = conn.prepareStatement( + "UPDATE " + TABLE_NAME + " SET leid=?, nm=?, rnd=? WHERE id=?")) { + for (int i = 0; i < k; i++) { + long id = Math.abs(RND.nextLong()) % ROW_COUNT; + upd.setString(1, "REPL-" + id); upd.setString(2, "Upd-" + id); + upd.setLong(3, RND.nextLong()); upd.setLong(4, id); + upd.addBatch(); + if ((i + 1) % BATCH_SIZE == 0) upd.executeBatch(); + } + upd.executeBatch(); + } + + // Measure WAL delta + ResultSet rs2 = stmt.executeQuery( + "SELECT pg_wal_lsn_diff(pg_current_wal_lsn(), '" + beforeLsn + "'::pg_lsn) AS wal_bytes"); + rs2.next(); + return rs2.getLong("wal_bytes"); + } + } + + static void printReplicationComparison(List results) { + System.out.println(); + System.out.println("╔══════════════════════╦═══════════╦══════════════╦════════════╦══════════╦═══════════╗"); + System.out.println("║ Database / scenario ║ k ║ payload KB ║ B/change ║ full MB ║ payload% ║"); + System.out.println("╠══════════════════════╬═══════════╬══════════════╬════════════╬══════════╬═══════════╣"); + String lastDb = ""; + for (ReplicationResult r : results) { + if (!r.db().equals(lastDb)) { + if (!lastDb.isEmpty()) + System.out.println("╠══════════════════════╬═══════════╬══════════════╬════════════╬══════════╬═══════════╣"); + lastDb = r.db(); + } + System.out.printf("║ %-20s ║ %,9d ║ %,12.1f ║ %,10.0f ║ %8.1f ║ %9.3f ║%n", + r.db(), r.k(), r.payloadKB(), r.bytesPerChange(), + r.fullTableMB(), r.payloadPct()); + } + System.out.println("╚══════════════════════╩═══════════╩══════════════╩════════════╩══════════╩═══════════╝"); + System.out.println(); + System.out.println(" payload% = (sync payload) / (full table scan) × 100"); + System.out.println(" Convex: O(k·log n) — only new/changed Merkle nodes transferred."); + System.out.println(" Content-addressing: unchanged sub-trees cost zero bandwidth."); + System.out.println(" MariaDB+changelog: O(k·row) — estimated from avg row size (requires binlog/WAL)."); + System.out.println(" MariaDB no-log: O(n·row) — full table scan needed without a changelog."); + System.out.println(" PostgreSQL: O(k·row) — WAL measured via pg_wal_lsn_diff."); + } + + // ── Result persistence ──────────────────────────────────────────────────── + + static String gitHash() { + try { + Process p = Runtime.getRuntime().exec(new String[]{"git", "rev-parse", "--short", "HEAD"}); + return new String(p.getInputStream().readAllBytes()).trim(); + } catch (Exception e) { + return "unknown"; + } + } + + /** + * Saves all results to bench-results/-.json under STORE_DIR. + * Format: one JSON object per line — header line first, then one result per line. + */ + static File saveResults(List results) throws IOException { + String hash = gitHash(); + String ts = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd-HHmmss")); + File dir = new File(STORE_DIR, "bench-results"); + dir.mkdirs(); + File file = new File(dir, ts + "-" + hash + ".json"); + + StringBuilder sb = new StringBuilder(); + // Header line + sb.append(String.format("{\"timestamp\":\"%s\",\"gitHash\":\"%s\",\"rows\":%d}%n", ts, hash, ROW_COUNT)); + // One result per line + for (BenchResult r : results) { + sb.append(String.format( + "{\"label\":\"%s\",\"rows\":%d,\"elapsedNs\":%d," + + "\"writtenBytes\":%d,\"allocatedBytes\":%d,\"compressedBytes\":%d,\"heapDeltaBytes\":%d," + + "\"dbRowsBefore\":%d,\"dbRowsAfter\":%d}%n", + r.label(), r.rows(), r.elapsedNs(), + r.writtenBytes(), r.allocatedBytes(), r.compressedBytes(), r.heapDeltaBytes(), + r.dbRowsBefore(), r.dbRowsAfter())); + } + Files.writeString(file.toPath(), sb.toString()); + return file; + } + + /** Loads a saved results file into a map keyed by label. */ + static Map loadResults(File file) throws IOException { + Map map = new LinkedHashMap<>(); + Pattern p = Pattern.compile("\"(\\w+)\":(-?\\d+|\"[^\"]*\")"); + for (String line : Files.readAllLines(file.toPath())) { + if (!line.contains("\"label\"")) continue; // skip header line + Matcher m = p.matcher(line); + Map fields = new LinkedHashMap<>(); + while (m.find()) fields.put(m.group(1), m.group(2).replace("\"", "")); + long dbRowsBefore = fields.containsKey("dbRowsBefore") ? Long.parseLong(fields.get("dbRowsBefore")) : -1; + long dbRowsAfter = fields.containsKey("dbRowsAfter") ? Long.parseLong(fields.get("dbRowsAfter")) : -1; + BenchResult r = new BenchResult( + fields.get("label"), + Integer.parseInt(fields.get("rows")), + Long.parseLong(fields.get("elapsedNs")), + Long.parseLong(fields.get("writtenBytes")), + Long.parseLong(fields.get("allocatedBytes")), + Long.parseLong(fields.get("compressedBytes")), + Long.parseLong(fields.get("heapDeltaBytes")), + dbRowsBefore, dbRowsAfter); + map.put(r.label(), r); + } + return map; + } + + /** Loads the header fields (timestamp, gitHash) from a saved file. */ + static String loadHeader(File file) throws IOException { + String header = Files.readAllLines(file.toPath()).get(0); + Pattern p = Pattern.compile("\"(timestamp|gitHash)\":\"([^\"]+)\""); + Matcher m = p.matcher(header); + Map h = new LinkedHashMap<>(); + while (m.find()) h.put(m.group(1), m.group(2)); + return h.getOrDefault("timestamp", "?") + " git:" + h.getOrDefault("gitHash", "?"); + } + + /** + * Compares two saved result files and prints a regression table. + * Positive Δ% = current is faster/smaller (improvement). + * Negative Δ% = current is slower/larger (regression). + */ + static void compareResultFiles(File baseline, File current) throws IOException { + Map base = loadResults(baseline); + Map curr = loadResults(current); + System.out.println("=== Benchmark Comparison ==="); + System.out.println(" Baseline : " + loadHeader(baseline) + " (" + baseline.getName() + ")"); + System.out.println(" Current : " + loadHeader(current) + " (" + current.getName() + ")"); + System.out.println(); + System.out.println("╔══════════════════════════════╦══════════════╦══════════════╦════════╦══════════╦══════════╦════════╗"); + System.out.println("║ Benchmark ║ base rows/s ║ cur rows/s ║ Δ% ║base B/row║ cur B/row║ Δ% ║"); + System.out.println("╠══════════════════════════════╬══════════════╬══════════════╬════════╬══════════╬══════════╬════════╣"); + + boolean anyRegression = false; + for (String label : base.keySet()) { + BenchResult b = base.get(label); + BenchResult c = curr.get(label); + if (c == null) { + System.out.printf("║ %-28s ║ %,12.0f ║ %12s ║ %6s ║ %8s ║ %8s ║ %6s ║%n", + label, b.rowsPerSec(), "MISSING", "-", "-", "-", "-"); + continue; + } + double throughputDelta = (c.rowsPerSec() - b.rowsPerSec()) / b.rowsPerSec() * 100; + String storageDeltaStr = "-"; + double storageDelta = Double.NaN; + if (b.writtenBytes() > 0 && c.writtenBytes() > 0) { + storageDelta = (c.writtenPerRow() - b.writtenPerRow()) / b.writtenPerRow() * 100; + storageDeltaStr = String.format("%+.1f%%", storageDelta); + } + // Flag regressions: throughput down >5% or storage up >5% + boolean regression = throughputDelta < -5.0 || storageDelta > 5.0; + if (regression) anyRegression = true; + System.out.printf("║ %-28s ║ %,12.0f ║ %,12.0f ║ %s ║ %8.0f ║ %8.0f ║ %s ║%n", + label, + b.rowsPerSec(), c.rowsPerSec(), + coloured(throughputDelta, true), + b.writtenPerRow() < 0 ? Double.NaN : b.writtenPerRow(), + c.writtenPerRow() < 0 ? Double.NaN : c.writtenPerRow(), + storageDeltaStr); + } + // Show new benchmarks that weren't in baseline + for (String label : curr.keySet()) { + if (!base.containsKey(label)) { + BenchResult c = curr.get(label); + System.out.printf("║ %-28s ║ %12s ║ %,12.0f ║ %6s ║ %8s ║ %8.0f ║ %6s ║%n", + label, "NEW", c.rowsPerSec(), "-", "-", + c.writtenPerRow() < 0 ? Double.NaN : c.writtenPerRow(), "-"); + } + } + System.out.println("╚══════════════════════════════╩══════════════╩══════════════╩════════╩══════════╩══════════╩════════╝"); + System.out.println(); + if (anyRegression) + System.out.println(" *** REGRESSIONS DETECTED (>5% throughput drop or >5% storage increase) ***"); + else + System.out.println(" No significant regressions detected."); + + // ── Memory usage comparison ────────────────────────────────────── + boolean anyHeapRow = false; + for (String label : base.keySet()) { + BenchResult b = base.get(label); + BenchResult c = curr.get(label); + if (c != null && b.heapDeltaBytes() > 0 && c.heapDeltaBytes() > 0) { anyHeapRow = true; break; } + } + if (anyHeapRow) { + System.out.println(); + System.out.println("=== Memory usage (heap delta) ==="); + System.out.println("╔══════════════════════════════╦══════════════╦══════════════╦════════╦══════════╦══════════╦════════╗"); + System.out.println("║ Benchmark ║ base MB ║ cur MB ║ Δ% ║base B/row║ cur B/row║ Δ% ║"); + System.out.println("╠══════════════════════════════╬══════════════╬══════════════╬════════╬══════════╬══════════╬════════╣"); + boolean anyHeapRegression = false; + for (String label : base.keySet()) { + BenchResult b = base.get(label); + BenchResult c = curr.get(label); + if (c == null || b.heapDeltaBytes() <= 0 || c.heapDeltaBytes() <= 0) continue; + double heapDelta = (c.heapDeltaBytes() - b.heapDeltaBytes()) / (double) b.heapDeltaBytes() * 100; + double bPerRow = (double) b.heapDeltaBytes() / b.rows(); + double cPerRow = (double) c.heapDeltaBytes() / c.rows(); + if (heapDelta > 5.0) anyHeapRegression = true; + String deltaStr = String.format("%+6.1f%%", heapDelta); + System.out.printf("║ %-28s ║ %,12.1f ║ %,12.1f ║ %s ║ %8.0f ║ %8.0f ║ %s ║%n", + label, + b.heapDeltaMB(), c.heapDeltaMB(), + deltaStr, + bPerRow, cPerRow, + deltaStr); + } + // New benchmarks not in baseline + for (String label : curr.keySet()) { + if (base.containsKey(label)) continue; + BenchResult c = curr.get(label); + if (c.heapDeltaBytes() <= 0) continue; + double cPerRow = (double) c.heapDeltaBytes() / c.rows(); + System.out.printf("║ %-28s ║ %12s ║ %,12.1f ║ %6s ║ %8s ║ %8.0f ║ %6s ║%n", + label, "NEW", c.heapDeltaMB(), "-", "-", cPerRow, "-"); + } + System.out.println("╚══════════════════════════════╩══════════════╩══════════════╩════════╩══════════╩══════════╩════════╝"); + if (anyHeapRegression) + System.out.println(" *** HEAP REGRESSIONS (>5% increase) ***"); + } + + // ── Row-count comparison ───────────────────────────────────────── + boolean anyRowIssue = false; + StringBuilder rowNotes = new StringBuilder(); + for (String label : curr.keySet()) { + BenchResult c = curr.get(label); + if (c.dbRowsAfter() < 0) continue; + BenchResult b = base.get(label); + // Flag if DB had accumulated rows going in (Etch store not cleared) + if (c.dbRowsBefore() > 0) { + rowNotes.append(String.format(" [ACCUMULATED] %-30s before=%,d after=%,d (Etch store not cleared)%n", + label, c.dbRowsBefore(), c.dbRowsAfter())); + anyRowIssue = true; + } + // Flag if row delta doesn't match expected insert count + long delta = c.dbRowsAfter() - c.dbRowsBefore(); + if (c.dbRowsBefore() >= 0 && delta != c.rows()) { + rowNotes.append(String.format(" [MISMATCH] %-30s expected +%,d rows but got +%,d%n", + label, c.rows(), delta)); + anyRowIssue = true; + } + // Show change in final row count vs baseline + if (b != null && b.dbRowsAfter() >= 0) { + long rowChange = c.dbRowsAfter() - b.dbRowsAfter(); + if (rowChange != 0) { + rowNotes.append(String.format(" [ROW CHANGE] %-30s base after=%,d cur after=%,d Δ=%+,d%n", + label, b.dbRowsAfter(), c.dbRowsAfter(), rowChange)); + } + } + } + if (anyRowIssue || rowNotes.length() > 0) { + System.out.println(" Row count notes:"); + System.out.print(rowNotes); + } + } + + /** Formats a delta percentage as a fixed-width string. */ + static String coloured(double delta, boolean higherIsBetter) { + // No ANSI colours since output may be piped; use +/- symbols instead + char flag = (higherIsBetter ? delta >= 0 : delta <= 0) ? '+' : '!'; + return String.format("%c%5.1f%%", flag, delta); + } + + // ── Helpers ──────────────────────────────────────────────────────────────── + + /** + * Counts live rows in a Convex table, handling block-packed entries correctly. + */ + static long countConvexRows(SQLSchema schema) { + SQLTable table = schema.getLiveTable(TABLE_NAME); + if (table == null) return 0; + convex.core.data.Index rows = table.getRows(); + if (rows == null) return 0; + long[] count = {0}; + rows.forEach((k, v) -> { + if (RowBlock.isBlock(v)) { + int n = RowBlock.count(v); + for (int i = 0; i < n; i++) { + if (SQLRow.isLive(RowBlock.getRow(v, i))) count[0]++; + } + } else if (SQLRow.isLive((AVector)v)) { + count[0]++; + } + }); + return count[0]; + } + + /** Counts rows in the MariaDB benchmark table. Returns -1 on failure. */ + static long countMariaDBRows() { + String connUrl = MARIADB_URL + "&user=" + MARIADB_USER + "&password=" + MARIADB_PASS; + try (Connection conn = DriverManager.getConnection(connUrl); + Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("SELECT COUNT(*) FROM " + TABLE_NAME); + if (rs.next()) return rs.getLong(1); + } catch (Exception e) { + // table may not exist yet + } + return -1; + } + + /** Counts rows in the PostgreSQL benchmark table. Returns -1 on failure. */ + static long countPostgresRows() { + try (Connection conn = DriverManager.getConnection(PSQL_URL, PSQL_USER, PSQL_PASS); + Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("SELECT COUNT(*) FROM " + TABLE_NAME); + if (rs.next()) return rs.getLong(1); + } catch (Exception e) { + // table may not exist yet + } + return -1; + } + + /** + * Finds the most recent previous result file and prints a comparison against it. + * Called automatically after every run. + */ + static void autoCompare(File currentFile) { + try { + File dir = currentFile.getParentFile(); + File[] files = dir.listFiles((d, n) -> n.endsWith(".json")); + if (files == null || files.length < 2) { + System.out.println("\nNo previous results to auto-compare against."); + return; + } + java.util.Arrays.sort(files, java.util.Comparator.comparing(File::getName)); + File baseline = null; + for (int i = files.length - 1; i >= 0; i--) { + if (!files[i].getName().equals(currentFile.getName())) { + baseline = files[i]; + break; + } + } + if (baseline == null) { + System.out.println("\nNo previous results to auto-compare against."); + return; + } + System.out.println("\n=== Auto-comparison with previous run ==="); + compareResultFiles(baseline, currentFile); + } catch (Exception e) { + System.out.println("\nAuto-compare failed: " + e.getMessage()); + } + } + + static void createConvexTable(SQLSchema schema) { + schema.createTable(TABLE_NAME, + new String[] { "ID", "LEID", "NM", "RND" }, + new ConvexType[]{ ConvexType.INTEGER, ConvexType.VARCHAR, ConvexType.VARCHAR, ConvexType.INTEGER }); + } + + /** + * Queries MariaDB information_schema for on-disk table size (data + index). + * Runs ANALYZE TABLE first to refresh the statistics. + */ + static long mariaDBTableBytes() throws Exception { + String connUrl = MARIADB_URL + "&user=" + MARIADB_USER + "&password=" + MARIADB_PASS; + try (Connection conn = DriverManager.getConnection(connUrl); + Statement stmt = conn.createStatement()) { + stmt.execute("ANALYZE TABLE " + TABLE_NAME); + ResultSet rs = stmt.executeQuery( + "SELECT (DATA_LENGTH + INDEX_LENGTH) AS total_bytes " + + "FROM information_schema.TABLES " + + "WHERE TABLE_SCHEMA = '" + MARIADB_DB + "' AND TABLE_NAME = '" + TABLE_NAME + "'"); + if (rs.next()) return rs.getLong("total_bytes"); + } + return -1; + } + + /** + * Returns the used data bytes recorded in the Etch file header, excluding + * any pre-allocated (reserved) space at the end of the file. + * Etch stores the actual data length as a long at byte offset 4. + */ + static long etchDataBytes(File file) throws IOException { + try (RandomAccessFile raf = new RandomAccessFile(file, "r")) { + raf.seek(4); // OFFSET_FILE_SIZE = SIZE_HEADER_MAGIC(2) + SIZE_HEADER_VERSION(2) + return raf.readLong(); + } + } + + /** + * Estimates compressed file size by sampling up to 4 MB from the file, + * compressing with GZIP, and extrapolating the ratio to the full file size. + * Uses the Etch data length (not the physical file size) to avoid counting reserved space. + */ + static long estimateCompressedSize(File file) throws IOException { + long fileSize = etchDataBytes(file); + if (fileSize == 0) return 0; + + int sampleSize = (int) Math.min(fileSize, 4 * 1024 * 1024); + byte[] sample = new byte[sampleSize]; + try (FileInputStream fis = new FileInputStream(file)) { + int read = 0; + while (read < sampleSize) { + int n = fis.read(sample, read, sampleSize - read); + if (n < 0) break; + read += n; + } + } + + ByteArrayOutputStream baos = new ByteArrayOutputStream(sampleSize / 2); + try (GZIPOutputStream gz = new GZIPOutputStream(baos)) { + gz.write(sample); + } + double ratio = (double) baos.size() / sampleSize; + return (long) (fileSize * ratio); + } + + static long usedHeap() { + Runtime rt = Runtime.getRuntime(); + return rt.totalMemory() - rt.freeMemory(); + } + + // ── Comparison table ────────────────────────────────────────────────────── + + /** + * Returns true if a benchmark label belongs to the "versioned" group. + * Versioned = includes history tracking (MariaDB WITH SYSTEM VERSIONING, Convex VersionedSQLSchema). + */ + static boolean isVersioned(String label) { + return label.contains("versioned") || label.contains("Versioned"); + } + + static void printComparison(List results) { + System.out.println(); + // Storage columns: + // written MB = actual data bytes (etchDataBytes for Etch; information_schema for MariaDB) + // alloc MB = OS file size for Etch (includes pre-allocated pages); same as written for MariaDB + // ~gz/row = estimated gzip of written bytes / row count + String header = "╔══════════════════════════════╦══════════════╦═══════════╦══════════╦══════════╦══════════╦══════════╦══════════════╗"; + String cols = "║ Benchmark ║ rows/sec ║written MB ║ alloc MB ║B/row(wr) ║ ~gz/row ║ heapΔ MB ║ DB rows ║"; + String sep = "╠══════════════════════════════╬══════════════╬═══════════╬══════════╬══════════╬══════════╬══════════╬══════════════╣"; + String foot = "╚══════════════════════════════╩══════════════╩═══════════╩══════════╩══════════╩══════════╩══════════╩══════════════╝"; + + // Print plain group first, then versioned group + boolean anyAccumulated = false; + for (boolean versionedGroup : new boolean[]{false, true}) { + List group = results.stream() + .filter(r -> isVersioned(r.label()) == versionedGroup) + .toList(); + if (group.isEmpty()) continue; + + System.out.println(header); + System.out.println(cols); + System.out.println(sep); + String groupLabel = versionedGroup ? "VERSIONED (with history)" : "PLAIN (no history)"; + System.out.printf("║ %-72s ║%n", " ── " + groupLabel + " ──"); + System.out.println(sep); + for (BenchResult r : group) { + String dbRows; + if (r.dbRowsBefore() >= 0 && r.dbRowsAfter() >= 0) { + boolean accumulated = r.dbRowsBefore() > 0; + if (accumulated) anyAccumulated = true; + dbRows = String.format("%,d→%,d%s", r.dbRowsBefore(), r.dbRowsAfter(), accumulated ? "*" : " "); + } else { + dbRows = "-"; + } + System.out.printf("║ %-28s ║ %,12.0f ║ %9.1f ║ %8.1f ║ %8.0f ║ %8.0f ║ %+8.1f ║ %-12s ║%n", + r.label(), + r.rowsPerSec(), + r.writtenMB() < 0 ? Double.NaN : r.writtenMB(), + r.allocatedMB() < 0 ? Double.NaN : r.allocatedMB(), + r.writtenPerRow() < 0 ? Double.NaN : r.writtenPerRow(), + r.compPerRow() < 0 ? Double.NaN : r.compPerRow(), + r.heapDeltaMB(), + dbRows); + } + System.out.println(foot); + System.out.println(); + } + if (anyAccumulated) + System.out.println(" * DB had pre-existing rows (Etch store not cleared before run — storage metrics include prior data)."); + + // Ratio summary: each Convex variant vs best plain MariaDB result + BenchResult mariaBest = results.stream() + .filter(r -> r.label().startsWith("MariaDB") && !isVersioned(r.label())) + .reduce((a, b) -> a.rowsPerSec() > b.rowsPerSec() ? a : b).orElse(null); + List convexResults = results.stream().filter(r -> r.label().startsWith("Convex")).toList(); + if (mariaBest != null && !convexResults.isEmpty()) { + System.out.println(" Ratios vs best plain MariaDB:"); + for (BenchResult c : convexResults) { + System.out.printf(" %-30s throughput: %+.2fx", c.label(), c.rowsPerSec() / mariaBest.rowsPerSec()); + if (c.writtenBytes() > 0 && mariaBest.writtenBytes() > 0) { + System.out.printf(" storage(written): %.1fx storage(~gz): %.1fx", + (double) c.writtenBytes() / mariaBest.writtenBytes(), + (double) c.compressedBytes() / mariaBest.writtenBytes()); + } + System.out.println(); + } + } + } + + static void printSingleRowComparison(List results) { + System.out.println("╔══════════════════════════════╦══════════════╦══════════════╗"); + System.out.println("║ Benchmark ║ avg µs/row ║ rows/sec ║"); + System.out.println("╠══════════════════════════════╬══════════════╬══════════════╣"); + for (BenchResult r : results) { + double usPerRow = r.elapsedNs() / 1000.0 / r.rows(); + System.out.printf("║ %-28s ║ %,12.1f ║ %,12.0f ║%n", r.label(), usPerRow, r.rowsPerSec()); + } + System.out.println("╚══════════════════════════════╩══════════════╩══════════════╝"); + + BenchResult convexDirect = results.stream().filter(r -> r.label().contains("direct")).findFirst().orElse(null); + BenchResult mariaJdbc = results.stream().filter(r -> r.label().contains("MariaDB JDBC")).findFirst().orElse(null); + if (convexDirect != null && mariaJdbc != null) { + System.out.printf("%n Convex direct vs MariaDB JDBC: %.1fx faster per row%n", + (double) mariaJdbc.elapsedNs() / convexDirect.elapsedNs()); + } + } +} diff --git a/convex-db/src/test/java/convex/db/sql/EtchCompactionDemo.java b/convex-db/src/test/java/convex/db/sql/EtchCompactionDemo.java new file mode 100644 index 000000000..17b46b6b1 --- /dev/null +++ b/convex-db/src/test/java/convex/db/sql/EtchCompactionDemo.java @@ -0,0 +1,261 @@ +package convex.db.sql; + +import java.io.File; +import java.io.RandomAccessFile; + +import convex.core.data.Vectors; +import convex.core.data.prim.CVMLong; +import convex.db.ConvexDB; +import convex.db.calcite.ConvexType; +import convex.db.lattice.SQLDatabase; +import convex.db.lattice.SQLSchema; +import convex.db.lattice.VersionedSQLSchema; +import convex.etch.EtchStore; +import convex.node.NodeServer; + +/** + * Demonstrates Etch GC compaction for both regular and versioned SQL tables. + * + *

Etch is append-only. Every write touches ~depth Index nodes in the radix + * tree; each touched node becomes a new cell while the old version remains as + * a dead cell. Compaction copies only the reachable cells to a fresh file. + * + *

Two scenarios are compared: + *

    + *
  • Regular table: updates overwrite rows; dead cells = old path nodes only
    • + *
  • Versioned table: history entries accumulate (never deleted), so the live + * set is larger but dead path-node churn is similar per write
    • + *
+ * + * Compaction is required to remove dead index nodes + * It is also achieved by creating a fresh replica node + * + */ +public class EtchCompactionDemo { + + static final int ROW_COUNT = 200_000; + static final int UPDATE_PASSES = 5; // full-table update passes after initial insert + static final int PERSIST_EVERY = 1_000; // force setRootData every N rows to generate dead cells + static final String DB_NAME = "bench"; + static final String TABLE_NAME = "t"; + static final String STORE_DIR = "/home/rob/etch/"; + + public static void main(String[] args) throws Exception { + runRegular(); + System.out.println(); + runVersioned(); + } + + // ── Regular table ───────────────────────────────────────────────────── + + static void runRegular() throws Exception { + System.out.println("=== Regular table compaction ==="); + File srcFile = new File(STORE_DIR, "compaction-src.etch"); + File dstFile = new File(STORE_DIR, "compaction-dst.etch"); + + // ── Phase 1: build a bloated store ──────────────────────────────── + System.out.printf("Building store: %,d rows → %s%n", ROW_COUNT, srcFile); + + if (srcFile.exists()) srcFile.delete(); + if (dstFile.exists()) dstFile.delete(); + + EtchStore srcStore = EtchStore.create(srcFile); + NodeServer server = ConvexDB.createNodeServer(srcStore); + server.launch(); + ConvexDB cdb = ConvexDB.connect(server.getCursor()); + + SQLDatabase db = cdb.database(DB_NAME); + SQLSchema schema = db.tables(); + schema.createTable(TABLE_NAME, new String[]{"id", "val1", "val2"}, + new ConvexType[]{ConvexType.INTEGER, ConvexType.INTEGER, ConvexType.INTEGER}); + + long insertStart = System.nanoTime(); + for (int i = 0; i < ROW_COUNT; i++) { + schema.insert(TABLE_NAME, Vectors.of(CVMLong.create(i), CVMLong.create(i * 2L), CVMLong.create(i * 3L))); + if ((i + 1) % PERSIST_EVERY == 0) server.persistSnapshot(server.getLocalValue()); + } + long insertMs = (System.nanoTime() - insertStart) / 1_000_000; + System.out.printf(" Inserted %,d rows in %,d ms (%.0f rows/s)%n", + ROW_COUNT, insertMs, ROW_COUNT / (insertMs / 1000.0)); + + System.out.printf(" Updating all rows %d times to generate dead cells...%n", UPDATE_PASSES); + long updateStart = System.nanoTime(); + for (int pass = 1; pass <= UPDATE_PASSES; pass++) { + for (int i = 0; i < ROW_COUNT; i++) { + schema.insert(TABLE_NAME, Vectors.of(CVMLong.create(i), + CVMLong.create(i * 2L + pass), CVMLong.create(i * 3L + pass))); + if ((i + 1) % PERSIST_EVERY == 0) server.persistSnapshot(server.getLocalValue()); + } + } + long updateMs = (System.nanoTime() - updateStart) / 1_000_000; + System.out.printf(" Updated %,d rows × %d passes in %,d ms%n%n", + ROW_COUNT, UPDATE_PASSES, updateMs); + + server.persistSnapshot(server.getLocalValue()); + server.close(); + srcStore.close(); + + long srcBytes = etchDataBytes(srcFile); + System.out.printf(" Etch size before compaction: %,.1f MB (%,d B/row)%n%n", + srcBytes / 1_048_576.0, srcBytes / ROW_COUNT); + + // ── Phase 2: compact ────────────────────────────────────────────── + System.out.printf("Compacting → %s%n", dstFile); + EtchStore srcReopen = EtchStore.create(srcFile); + long compactStart = System.nanoTime(); + EtchStore dstStore = srcReopen.compact(dstFile); + long compactMs = (System.nanoTime() - compactStart) / 1_000_000; + srcReopen.close(); + + long dstBytes = etchDataBytes(dstFile); + double ratio = (double) srcBytes / dstBytes; + System.out.printf(" Compaction done in %,d ms%n", compactMs); + System.out.printf(" Etch size after: %,.1f MB (%,d B/row)%n", dstBytes / 1_048_576.0, dstBytes / ROW_COUNT); + System.out.printf(" Compaction ratio: %.1f×%n%n", ratio); + + // ── Phase 3: verify ─────────────────────────────────────────────── + System.out.println("Verifying data integrity in compacted store..."); + NodeServer server2 = ConvexDB.createNodeServer(dstStore); + server2.launch(); + ConvexDB cdb2 = ConvexDB.connect(server2.getCursor()); + SQLSchema schema2 = cdb2.database(DB_NAME).tables(); + + long rowCount = schema2.getRowCount(TABLE_NAME); + System.out.printf(" Row count: %,d%n", rowCount); + + int errors = 0; + for (int i : new int[]{0, 1, ROW_COUNT / 2, ROW_COUNT - 1}) { + if (schema2.selectByKey(TABLE_NAME, CVMLong.create(i)) == null) { + System.out.printf(" ERROR: row %d missing!%n", i); + errors++; + } + } + System.out.printf(" Spot checks: %s%n", errors == 0 ? "PASS" : errors + " FAILURES"); + server2.close(); + dstStore.close(); + + printSummary(srcBytes, dstBytes, ratio, rowCount); + } + + // ── Versioned table ─────────────────────────────────────────────────── + + static void runVersioned() throws Exception { + System.out.println("=== Versioned table compaction ==="); + File srcFile = new File(STORE_DIR, "compaction-versioned-src.etch"); + File dstFile = new File(STORE_DIR, "compaction-versioned-dst.etch"); + + // ── Phase 1: build a bloated store ──────────────────────────────── + System.out.printf("Building store: %,d rows → %s%n", ROW_COUNT, srcFile); + + if (srcFile.exists()) srcFile.delete(); + if (dstFile.exists()) dstFile.delete(); + + EtchStore srcStore = EtchStore.create(srcFile); + NodeServer server = ConvexDB.createNodeServer(srcStore); + server.launch(); + ConvexDB cdb = ConvexDB.connect(server.getCursor()); + + SQLDatabase db = cdb.database(DB_NAME); + VersionedSQLSchema schema = VersionedSQLSchema.wrap(db.tables()); + schema.createTable(TABLE_NAME, new String[]{"id", "val1", "val2"}, + new ConvexType[]{ConvexType.INTEGER, ConvexType.INTEGER, ConvexType.INTEGER}); + + long insertStart = System.nanoTime(); + for (int i = 0; i < ROW_COUNT; i++) { + schema.insert(TABLE_NAME, Vectors.of(CVMLong.create(i), CVMLong.create(i * 2L), CVMLong.create(i * 3L))); + if ((i + 1) % PERSIST_EVERY == 0) server.persistSnapshot(server.getLocalValue()); + } + long insertMs = (System.nanoTime() - insertStart) / 1_000_000; + System.out.printf(" Inserted %,d rows in %,d ms (%.0f rows/s)%n", + ROW_COUNT, insertMs, ROW_COUNT / (insertMs / 1000.0)); + + System.out.printf(" Updating all rows %d times (history entries accumulate)...%n", UPDATE_PASSES); + long updateStart = System.nanoTime(); + for (int pass = 1; pass <= UPDATE_PASSES; pass++) { + for (int i = 0; i < ROW_COUNT; i++) { + schema.insert(TABLE_NAME, Vectors.of(CVMLong.create(i), + CVMLong.create(i * 2L + pass), CVMLong.create(i * 3L + pass))); + if ((i + 1) % PERSIST_EVERY == 0) server.persistSnapshot(server.getLocalValue()); + } + } + long updateMs = (System.nanoTime() - updateStart) / 1_000_000; + System.out.printf(" Updated %,d rows × %d passes in %,d ms%n%n", + ROW_COUNT, UPDATE_PASSES, updateMs); + + server.persistSnapshot(server.getLocalValue()); + server.close(); + srcStore.close(); + + long srcBytes = etchDataBytes(srcFile); + System.out.printf(" Etch size before compaction: %,.1f MB (%,d B/row)%n%n", + srcBytes / 1_048_576.0, srcBytes / ROW_COUNT); + + // ── Phase 2: compact ────────────────────────────────────────────── + System.out.printf("Compacting → %s%n", dstFile); + EtchStore srcReopen = EtchStore.create(srcFile); + long compactStart = System.nanoTime(); + EtchStore dstStore = srcReopen.compact(dstFile); + long compactMs = (System.nanoTime() - compactStart) / 1_000_000; + srcReopen.close(); + + long dstBytes = etchDataBytes(dstFile); + double ratio = (double) srcBytes / dstBytes; + System.out.printf(" Compaction done in %,d ms%n", compactMs); + System.out.printf(" Etch size after: %,.1f MB (%,d B/row)%n", dstBytes / 1_048_576.0, dstBytes / ROW_COUNT); + System.out.printf(" Compaction ratio: %.1f×%n%n", ratio); + + // ── Phase 3: verify ─────────────────────────────────────────────── + System.out.println("Verifying data integrity in compacted store..."); + NodeServer server2 = ConvexDB.createNodeServer(dstStore); + server2.launch(); + ConvexDB cdb2 = ConvexDB.connect(server2.getCursor()); + VersionedSQLSchema schema2 = VersionedSQLSchema.wrap(cdb2.database(DB_NAME).tables()); + + long rowCount = schema2.getRowCount(TABLE_NAME); + System.out.printf(" Live row count: %,d%n", rowCount); + + int errors = 0; + for (int i : new int[]{0, 1, ROW_COUNT / 2, ROW_COUNT - 1}) { + var row = schema2.selectByKey(TABLE_NAME, CVMLong.create(i)); + if (row == null) { + System.out.printf(" ERROR: row %d missing!%n", i); + errors++; + } + // Verify history: expect 1 insert + UPDATE_PASSES updates = UPDATE_PASSES+1 entries + var history = schema2.getHistory(TABLE_NAME, CVMLong.create(i)); + if (history.size() != UPDATE_PASSES + 1) { + System.out.printf(" ERROR: row %d history size %d (expected %d)!%n", + i, history.size(), UPDATE_PASSES + 1); + errors++; + } + } + System.out.printf(" Spot checks (rows + history): %s%n", errors == 0 ? "PASS" : errors + " FAILURES"); + server2.close(); + dstStore.close(); + + long historyEntries = (long) ROW_COUNT * (UPDATE_PASSES + 1); + System.out.printf(" Total history entries preserved: %,d%n", historyEntries); + printSummary(srcBytes, dstBytes, ratio, rowCount); + } + + // ── Helpers ─────────────────────────────────────────────────────────── + + static void printSummary(long srcBytes, long dstBytes, double ratio, long rowCount) { + System.out.println(); + System.out.println("=== Summary ==="); + System.out.printf(" Before compaction: %,.1f MB%n", srcBytes / 1_048_576.0); + System.out.printf(" After compaction: %,.1f MB%n", dstBytes / 1_048_576.0); + System.out.printf(" Space saved: %,.1f MB (%.1f%% of original)%n", + (srcBytes - dstBytes) / 1_048_576.0, 100.0 * (srcBytes - dstBytes) / srcBytes); + System.out.printf(" Compaction ratio: %.1f×%n", ratio); + System.out.printf(" Live rows: %,d%n", rowCount); + } + + /** Reads the Etch data length field from offset 4 (past magic + version bytes). */ + static long etchDataBytes(File file) throws Exception { + try (RandomAccessFile raf = new RandomAccessFile(file, "r")) { + raf.seek(4); // OFFSET_FILE_SIZE = SIZE_HEADER_MAGIC(2) + SIZE_HEADER_VERSION(2) + return raf.readLong(); + } + } +} diff --git a/convex-db/src/test/java/convex/db/sql/EtchPKLookupBench.java b/convex-db/src/test/java/convex/db/sql/EtchPKLookupBench.java index 733cd4a2d..67f092551 100644 --- a/convex-db/src/test/java/convex/db/sql/EtchPKLookupBench.java +++ b/convex-db/src/test/java/convex/db/sql/EtchPKLookupBench.java @@ -1,5 +1,11 @@ package convex.db.sql; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Arrays; import java.util.Random; import convex.core.data.AVector; @@ -67,27 +73,75 @@ static void runBenchmark(int rowCount) throws Exception { // Random PK lookups Random rng = new Random(42); - long totalNs = 0; + long[] timings = new long[LOOKUPS]; long maxNs = 0; + int maxIter = 0; for (int i = 0; i < LOOKUPS; i++) { int key = rng.nextInt(rowCount); long start = System.nanoTime(); AVector row = tables.selectByKey("t", CVMLong.create(key)); long elapsed = System.nanoTime() - start; if (row == null) throw new AssertionError("Missing row for key=" + key); - totalNs += elapsed; - if (elapsed > maxNs) maxNs = elapsed; + timings[i] = elapsed; + if (elapsed > maxNs) { maxNs = elapsed; maxIter = i; } } - - double avgUs = (totalNs / (double) LOOKUPS) / 1_000.0; - double maxUs = maxNs / 1_000.0; - System.out.printf(" PK lookups (%d) avg %,.1f us, max %,.1f us%n", LOOKUPS, avgUs, maxUs); + reportLookups("PK lookups", timings, maxIter); // Row count (should be O(1) with liveCount) t0 = System.nanoTime(); long count = tables.getRowCount("t"); report("getRowCount() = " + count, System.nanoTime() - t0); + // SQL queries via JDBC + cdb.register("bench"); + try (Connection conn = DriverManager.getConnection("jdbc:convex:database=bench")) { + + // Warmup SQL + try (Statement ws = conn.createStatement()) { + ws.executeQuery("SELECT * FROM t WHERE ID = 0").close(); + } + + // Random PK lookups: SELECT * FROM t WHERE ID = ? + rng = new Random(42); + timings = new long[LOOKUPS]; + maxNs = 0; + maxIter = 0; + try (PreparedStatement ps = conn.prepareStatement("SELECT * FROM t WHERE ID = ?")) { + for (int i = 0; i < LOOKUPS; i++) { + long key = rng.nextInt(rowCount); + ps.setLong(1, key); + long start = System.nanoTime(); + try (ResultSet rs = ps.executeQuery()) { + if (!rs.next()) throw new AssertionError("Missing row for key=" + key); + } + long elapsed = System.nanoTime() - start; + timings[i] = elapsed; + if (elapsed > maxNs) { maxNs = elapsed; maxIter = i; } + } + } + reportLookups("SQL WHERE ID=x", timings, maxIter); + + // Top-10 by ID descending: SELECT * FROM t ORDER BY ID DESC LIMIT 10 + timings = new long[LOOKUPS]; + maxNs = 0; + maxIter = 0; + try (PreparedStatement ps = conn.prepareStatement("SELECT * FROM t ORDER BY ID DESC LIMIT 10")) { + for (int i = 0; i < LOOKUPS; i++) { + long start = System.nanoTime(); + try (ResultSet rs = ps.executeQuery()) { + int rows = 0; + while (rs.next()) rows++; + if (rows != 10) throw new AssertionError("Expected 10 rows, got " + rows); + } + long elapsed = System.nanoTime() - start; + timings[i] = elapsed; + if (elapsed > maxNs) { maxNs = elapsed; maxIter = i; } + } + } + reportLookups("SQL ORDER BY ID DESC LIMIT 10", timings, maxIter); + } + cdb.unregister("bench"); + server.close(); store.close(); } @@ -96,4 +150,18 @@ static void report(String label, long elapsedNs) { double ms = elapsedNs / 1_000_000.0; System.out.printf(" %-25s %,.1f ms%n", label, ms); } + + static void reportLookups(String label, long[] timings, int maxIter) { + long totalNs = 0; + long maxNs = 0; + for (long t : timings) { totalNs += t; if (t > maxNs) maxNs = t; } + long[] sorted = timings.clone(); + Arrays.sort(sorted); + long medianNs = sorted[sorted.length / 2]; + double avgUs = (totalNs / (double) timings.length) / 1_000.0; + double medianUs = medianNs / 1_000.0; + double maxUs = maxNs / 1_000.0; + System.out.printf(" %-32s avg %,.1f us, median %,.1f us, max %,.1f us (iter %d)%n", + label + " (" + timings.length + ")", avgUs, medianUs, maxUs, maxIter); + } } diff --git a/convex-db/src/test/java/convex/db/sql/InsertDemo.java b/convex-db/src/test/java/convex/db/sql/InsertDemo.java index 40ca73341..90a3e5abb 100644 --- a/convex-db/src/test/java/convex/db/sql/InsertDemo.java +++ b/convex-db/src/test/java/convex/db/sql/InsertDemo.java @@ -1,12 +1,17 @@ package convex.db.sql; +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.Statement; -import convex.core.crypto.AKeyPair; +import convex.core.data.ACell; +import convex.core.data.AVector; import convex.core.data.Vectors; import convex.core.data.prim.CVMLong; @@ -14,39 +19,47 @@ import convex.db.calcite.ConvexType; import convex.db.lattice.SQLDatabase; +import convex.db.lattice.SQLSchema; +import convex.db.lattice.VersionedSQLSchema; +import convex.db.lattice.VersionedSQLTable; import convex.etch.EtchStore; import convex.node.NodeServer; /** * Insert performance demo/benchmark for Convex DB. * - * Uses a realistic setup: NodeServer with temp Etch persistence, - * SQLDatabase connected through the cursor chain. + * Uses a realistic setup: NodeServer with temp Etch persistence, SQLDatabase + * connected through the cursor chain. * - * Tests: - * 1. Standalone (no persistence, baseline) - * 2. Direct lattice insert via NodeServer - * 3. JDBC individual INSERT statements - * 4. JDBC PreparedStatement - * 5. JDBC PreparedStatement batch + * Tests: 1. Standalone (no persistence, baseline) 2. Direct lattice insert via + * NodeServer 3. JDBC individual INSERT statements 4. JDBC PreparedStatement 5. + * JDBC PreparedStatement batch * * Each NodeServer test syncs to Etch storage at the end. */ public class InsertDemo { - static final int ROW_COUNT = 10_000; + static final int ROW_COUNT = 200_000; + static final int SAMPLE_COUNT = 20; + static final int SAMPLE_INTERVAL = ROW_COUNT / SAMPLE_COUNT; static final String DB_NAME = "bench"; + static final String STORE_DIR = "/home/rob/etch/"; + static final Random RND = new Random(); public static void main(String[] args) throws Exception { System.out.println("=== Convex DB Insert Benchmark ==="); System.out.println("Row count: " + ROW_COUNT); - System.out.println("Setup: NodeServer + EtchStore (temp)\n"); + System.out.println("Setup: NodeServer + EtchStore (permanent, dir=" + STORE_DIR + ")\n"); benchmarkStandalone(); + System.gc(); benchmarkDirectLattice(); - benchmarkJdbcIndividual(); + System.gc(); + // benchmarkJdbcIndividual(); benchmarkJdbcPrepared(); + System.gc(); benchmarkJdbcPreparedBatch(); + System.gc(); System.out.println("\nDone."); } @@ -54,74 +67,116 @@ public static void main(String[] args) throws Exception { // ========== Benchmarks ========== /** - * Pure baseline: standalone SQLDatabase (no NodeServer, no persistence) + * Pure baseline: standalone VersionedSQLSchema (no NodeServer, no persistence + * Nanotime Values INSe) */ static void benchmarkStandalone() throws Exception { - AKeyPair kp = AKeyPair.generate(); - SQLDatabase db = SQLDatabase.create(DB_NAME, kp); - createTable(db); + VersionedSQLSchema schema = VersionedSQLSchema.create(); + createTable(schema); + List samples = new ArrayList<>(); + long memBefore = usedHeap(); long start = System.nanoTime(); for (int i = 0; i < ROW_COUNT; i++) { - db.tables().insert("t", Vectors.of(CVMLong.create(i), "LEID-" + i, "Name-" + i)); + schema.insert("t", Vectors.of(CVMLong.create(i), "LEID-" + i, "Name-" + i, CVMLong.create(RND.nextLong()))); + if ((i + 1) % SAMPLE_INTERVAL == 0) + samples.add(new long[] { i + 1, System.nanoTime() - start, usedHeap() - memBefore }); } + // Update pass: re-insert rows near id=100,000 with fresh random values → UPDATE + // history entries + for (int i = 99_995; i <= 100_005; i++) + schema.insert("t", Vectors.of(CVMLong.create(i), "LEID-" + i, "Name-" + i, CVMLong.create(RND.nextLong()))); long elapsed = System.nanoTime() - start; - - report("Standalone (no persist)", elapsed, ROW_COUNT); - verify(db, ROW_COUNT, "Standalone"); + long memAfter = usedHeap(); + + report("Standalone (no persist)", elapsed, ROW_COUNT, memAfter - memBefore); + printProgress(samples); + verify(schema, ROW_COUNT, "Standalone"); + printRowCount(schema, "before delete"); + schema.deleteByKey("t", CVMLong.create(100_000)); + printRowCount(schema, "after delete id=100,000"); + printHistory(schema, 100_000); } /** - * Direct SQLSchema.insert() via NodeServer cursor chain + * Direct VersionedSQLSchema.insert() via NodeServer cursor chain */ static void benchmarkDirectLattice() throws Exception { - EtchStore store = EtchStore.createTemp(); + File storeFile = new File(STORE_DIR, "direct.etch"); + EtchStore store = EtchStore.create(storeFile); NodeServer server = ConvexDB.createNodeServer(store); server.launch(); SQLDatabase db = SQLDatabase.connect(server.getCursor(), DB_NAME); - createTable(db); + VersionedSQLSchema schema = VersionedSQLSchema.wrap(db.tables()); + createTable(schema); + List samples = new ArrayList<>(); + long memBefore = usedHeap(); long start = System.nanoTime(); for (int i = 0; i < ROW_COUNT; i++) { - db.tables().insert("t", Vectors.of(CVMLong.create(i), "LEID-" + i, "Name-" + i)); + schema.insert("t", Vectors.of(CVMLong.create(i), "LEID-" + i, "Name-" + i, CVMLong.create(RND.nextLong()))); + if ((i + 1) % SAMPLE_INTERVAL == 0) + samples.add(new long[] { i + 1, System.nanoTime() - start, usedHeap() - memBefore }); } + // Update pass: re-insert rows near id=100,000 with fresh random values → UPDATE + // history entries + for (int i = 99_995; i <= 100_005; i++) + schema.insert("t", Vectors.of(CVMLong.create(i), "LEID-" + i, "Name-" + i, CVMLong.create(RND.nextLong()))); long elapsed = System.nanoTime() - start; + long memAfter = usedHeap(); - report("Direct lattice insert", elapsed, ROW_COUNT); + report("Direct lattice insert", elapsed, ROW_COUNT, memAfter - memBefore); + printProgress(samples); syncToStorage(server); - verify(db, ROW_COUNT, "Direct lattice"); + reportFileStats(storeFile, schema.getRowCount("t")); + verify(schema, ROW_COUNT, "Direct lattice"); + printRowCount(schema, "before delete"); + schema.deleteByKey("t", CVMLong.create(100_000)); + printRowCount(schema, "after delete id=100,000"); + printHistory(schema, 100_000); server.close(); store.close(); } /** - * Individual JDBC INSERT statements — full Calcite parse/plan/compile per statement + * Individual JDBC INSERT statements — full Calcite parse/plan/compile per + * statement */ static void benchmarkJdbcIndividual() throws Exception { - EtchStore store = EtchStore.createTemp(); + File storeFile = new File(STORE_DIR, "jdbc-individual.etch"); + EtchStore store = EtchStore.create(storeFile); NodeServer server = ConvexDB.createNodeServer(store); server.launch(); ConvexDB cdb = ConvexDB.connect(server.getCursor()); SQLDatabase db = cdb.database(DB_NAME); - createTable(db); + SQLSchema schema = db.tables(); + createTable(schema); cdb.register(DB_NAME); try (Connection conn = DriverManager.getConnection("jdbc:convex:database=" + DB_NAME); - Statement stmt = conn.createStatement()) { + Statement stmt = conn.createStatement()) { // Warm up: trigger Calcite class loading / JIT - stmt.executeUpdate("INSERT INTO t VALUES (-1, 'warm', 'up')"); + stmt.executeUpdate("INSERT INTO t VALUES (-1, 'warm', 'up', 0)"); + List samples = new ArrayList<>(); + long memBefore = usedHeap(); long start = System.nanoTime(); for (int i = 0; i < ROW_COUNT; i++) { - stmt.executeUpdate("INSERT INTO t VALUES (" + i + ", 'LEID-" + i + "', 'Name-" + i + "')"); + stmt.executeUpdate( + "INSERT INTO t VALUES (" + i + ", 'LEID-" + i + "', 'Name-" + i + "', " + RND.nextLong() + ")"); + if ((i + 1) % SAMPLE_INTERVAL == 0) + samples.add(new long[] { i + 1, System.nanoTime() - start, usedHeap() - memBefore }); } long elapsed = System.nanoTime() - start; + long memAfter = usedHeap(); - report("JDBC individual INSERT", elapsed, ROW_COUNT); + report("JDBC individual INSERT", elapsed, ROW_COUNT, memAfter - memBefore); + printProgress(samples); syncToStorage(server); } - verify(db, ROW_COUNT, "JDBC individual"); + reportFileStats(storeFile, schema.getRowCount("t")); + verify(schema, ROW_COUNT, "JDBC individual"); cdb.unregister(DB_NAME); server.close(); store.close(); @@ -131,30 +186,41 @@ static void benchmarkJdbcIndividual() throws Exception { * JDBC PreparedStatement — parse/plan/compile once, bind params each time */ static void benchmarkJdbcPrepared() throws Exception { - EtchStore store = EtchStore.createTemp(); + File storeFile = new File(STORE_DIR, "jdbc-prepared.etch"); + EtchStore store = EtchStore.create(storeFile); NodeServer server = ConvexDB.createNodeServer(store); server.launch(); ConvexDB cdb = ConvexDB.connect(server.getCursor()); SQLDatabase db = cdb.database(DB_NAME); - createTable(db); + SQLSchema schema = db.tables(); + createTable(schema); cdb.register(DB_NAME); try (Connection conn = DriverManager.getConnection("jdbc:convex:database=" + DB_NAME); - PreparedStatement ps = conn.prepareStatement("INSERT INTO t VALUES (?, ?, ?)")) { + PreparedStatement ps = conn.prepareStatement("INSERT INTO t VALUES (?, ?, ?, ?)")) { + List samples = new ArrayList<>(); + long memBefore = usedHeap(); long start = System.nanoTime(); for (int i = 0; i < ROW_COUNT; i++) { ps.setInt(1, i); ps.setString(2, "LEID-" + i); ps.setString(3, "Name-" + i); + ps.setLong(4, RND.nextLong()); ps.executeUpdate(); + if ((i + 1) % SAMPLE_INTERVAL == 0) + samples.add(new long[] { i + 1, System.nanoTime() - start, usedHeap() - memBefore }); } long elapsed = System.nanoTime() - start; + long memAfter = usedHeap(); - report("JDBC PreparedStatement", elapsed, ROW_COUNT); + report("JDBC PreparedStatement", elapsed, ROW_COUNT, memAfter - memBefore); + printProgress(samples); syncToStorage(server); } - verify(db, ROW_COUNT, "JDBC PreparedStatement"); + reportFileStats(storeFile, schema.getRowCount("t")); + verify(schema, ROW_COUNT, "JDBC PreparedStatement"); + printRowCount(schema, "total"); cdb.unregister(DB_NAME); server.close(); store.close(); @@ -164,31 +230,43 @@ static void benchmarkJdbcPrepared() throws Exception { * PreparedStatement batch — compile once, batch all parameter bindings */ static void benchmarkJdbcPreparedBatch() throws Exception { - EtchStore store = EtchStore.createTemp(); + File storeFile = new File(STORE_DIR, "jdbc-batch.etch"); + EtchStore store = EtchStore.create(storeFile); NodeServer server = ConvexDB.createNodeServer(store); server.launch(); ConvexDB cdb = ConvexDB.connect(server.getCursor()); SQLDatabase db = cdb.database(DB_NAME); - createTable(db); + SQLSchema schema = db.tables(); + createTable(schema); cdb.register(DB_NAME); try (Connection conn = DriverManager.getConnection("jdbc:convex:database=" + DB_NAME); - PreparedStatement ps = conn.prepareStatement("INSERT INTO t VALUES (?, ?, ?)")) { + PreparedStatement ps = conn.prepareStatement("INSERT INTO t VALUES (?, ?, ?, ?)")) { + List samples = new ArrayList<>(); + long memBefore = usedHeap(); long start = System.nanoTime(); for (int i = 0; i < ROW_COUNT; i++) { ps.setInt(1, i); ps.setString(2, "LEID-" + i); ps.setString(3, "Name-" + i); + ps.setLong(4, RND.nextLong()); ps.addBatch(); + if ((i + 1) % SAMPLE_INTERVAL == 0) { + ps.executeBatch(); + samples.add(new long[] { i + 1, System.nanoTime() - start, usedHeap() - memBefore }); + } } - ps.executeBatch(); long elapsed = System.nanoTime() - start; + long memAfter = usedHeap(); - report("JDBC PreparedStmt batch", elapsed, ROW_COUNT); + report("JDBC PreparedStmt batch", elapsed, ROW_COUNT, memAfter - memBefore); + printProgress(samples); syncToStorage(server); } - verify(db, ROW_COUNT, "JDBC PreparedStmt batch"); + reportFileStats(storeFile, schema.getRowCount("t")); + verify(schema, ROW_COUNT, "JDBC PreparedStmt batch"); + printRowCount(schema, "total"); cdb.unregister(DB_NAME); server.close(); store.close(); @@ -196,9 +274,9 @@ static void benchmarkJdbcPreparedBatch() throws Exception { // ========== Helpers ========== - static void createTable(SQLDatabase db) { - db.tables().createTable("t", new String[]{"ID", "LEID", "NM"}, - new ConvexType[]{ConvexType.INTEGER, ConvexType.VARCHAR, ConvexType.VARCHAR}); + static void createTable(SQLSchema schema) { + schema.createTable("t", new String[] { "ID", "LEID", "NM", "RND" }, + new ConvexType[] { ConvexType.INTEGER, ConvexType.VARCHAR, ConvexType.VARCHAR, ConvexType.INTEGER }); } static void syncToStorage(NodeServer server) throws Exception { @@ -209,11 +287,11 @@ static void syncToStorage(NodeServer server) throws Exception { } /** - * Smoke check: verifies row count via lattice API and JDBC SELECT COUNT, - * then spot-checks first (ID=0) and last (ID=ROW_COUNT-1) rows via JDBC. + * Smoke check: verifies row count via lattice API and JDBC SELECT COUNT, then + * spot-checks first (ID=0) and last (ID=ROW_COUNT-1) rows via JDBC. */ - static void verify(SQLDatabase db, int minExpectedRows, String label) throws Exception { - long latticeCount = db.tables().getRowCount("t"); + static void verify(SQLSchema schema, int minExpectedRows, String label) throws Exception { + long latticeCount = schema.getRowCount("t"); check(latticeCount >= minExpectedRows, label, "Lattice row count: expected >= " + minExpectedRows + ", got " + latticeCount); @@ -222,7 +300,7 @@ static void verify(SQLDatabase db, int minExpectedRows, String label) throws Exc ConvexDB vcdb = ConvexDB.lookup(DB_NAME); if (vcdb != null) { try (Connection conn = DriverManager.getConnection("jdbc:convex:database=" + DB_NAME); - Statement stmt = conn.createStatement()) { + Statement stmt = conn.createStatement()) { ResultSet rs = stmt.executeQuery("SELECT COUNT(*) AS cnt FROM t"); rs.next(); @@ -240,8 +318,7 @@ static void verify(SQLDatabase db, int minExpectedRows, String label) throws Exc int lastId = ROW_COUNT - 1; rs = stmt.executeQuery("SELECT ID, LEID, NM FROM t WHERE ID = " + lastId); check(rs.next(), label, "Row ID=" + lastId + " not found"); - check(("LEID-" + lastId).equals(rs.getString("LEID")), label, - "Row ID=" + lastId + " LEID mismatch"); + check(("LEID-" + lastId).equals(rs.getString("LEID")), label, "Row ID=" + lastId + " LEID mismatch"); } } System.out.println(" VERIFIED OK (" + latticeCount + " rows)"); @@ -253,9 +330,85 @@ static void check(boolean condition, String testLabel, String message) { } } - static void report(String label, long elapsedNs, int count) { + static void reportFileStats(File file, long rowCount) { + long fileBytes = file.length(); + double fileMB = fileBytes / (1024.0 * 1024.0); + double bytesPerRow = rowCount > 0 ? (double) fileBytes / rowCount : 0; + System.out.printf(" File: %-40s %,.1f MB (%,d rows = %.0f bytes/row)%n", file.getName(), fileMB, rowCount, + bytesPerRow); + } + + /** + * Prints a per-sample table: rows inserted, cumulative time, instantaneous + * rate, heap delta. + */ + static void printProgress(List samples) { + System.out.printf(" %-12s %-10s %-14s %-12s%n", "Rows", "Time(ms)", "Rate(rows/s)", "Heap delta"); + long prevRows = 0; + long prevNs = 0; + for (long[] s : samples) { + long rows = s[0]; + long totalNs = s[1]; + long heapBytes = s[2]; + long intervalRows = rows - prevRows; + long intervalNs = totalNs - prevNs; + double intervalRate = intervalRows / (intervalNs / 1_000_000_000.0); + double heapMB = heapBytes / (1024.0 * 1024.0); + System.out.printf(" %-,12d %-,10.0f %-,14.0f %+.1f MB%n", rows, totalNs / 1_000_000.0, intervalRate, + heapMB); + prevRows = rows; + prevNs = totalNs; + } + } + + static void printRowCount(SQLSchema schema, String label) { + long count = schema.getRowCount("t"); + System.out.printf(" Live rows in 't' (%s): %,d%n", label, count); + } + + static void printRowCount(VersionedSQLSchema schema, String label) { + long live = schema.getRowCount("t"); + VersionedSQLTable table = schema.getLiveTable("t"); + long history = table != null ? table.getHistoryIndex().count() : 0; + System.out.printf(" Live rows in 't' (%s): %,d (history entries: %,d)%n", label, live, history); + } + + static long usedHeap() { + Runtime rt = Runtime.getRuntime(); + return rt.totalMemory() - rt.freeMemory(); + } + + /** + * Prints the full history for a row in table "t" by primary key id. Shows each + * change event: type, nanotime, and values. + */ + static void printHistory(VersionedSQLSchema schema, long id) { + System.out.printf("%n --- History for id=%d ---%n", id); + List> history = schema.getHistory("t", CVMLong.create(id)); + if (history.isEmpty()) { + System.out.println(" (no history — id not found or JDBC path used)"); + return; + } + System.out.printf(" %-8s %-22s %s%n", "Type", "Nanotime", "Values"); + for (AVector entry : history) { + long ts = ((CVMLong) entry.get(1)).longValue(); + long ct = ((CVMLong) entry.get(2)).longValue(); + ACell vals = entry.get(0); + String type = switch ((int) ct) { + case (int) VersionedSQLTable.CT_INSERT -> "INSERT"; + case (int) VersionedSQLTable.CT_UPDATE -> "UPDATE"; + case (int) VersionedSQLTable.CT_DELETE -> "DELETE"; + default -> "UNKNOWN"; + }; + System.out.printf(" %-8s %-22d %s%n", type, ts, vals == null ? "DELETED" : vals); + } + System.out.println(); + } + + static void report(String label, long elapsedNs, int count, long heapDeltaBytes) { double ms = elapsedNs / 1_000_000.0; double rate = count / (ms / 1000.0); - System.out.printf("%-25s %,.0f ms (%,.0f rows/sec)%n", label, ms, rate); + double heapMB = heapDeltaBytes / (1024.0 * 1024.0); + System.out.printf("%-25s %,.0f ms (%,.0f rows/sec) heap delta: %+,.1f MB%n", label, ms, rate, heapMB); } } diff --git a/convex-db/src/test/java/convex/db/sql/ScalingBench.java b/convex-db/src/test/java/convex/db/sql/ScalingBench.java index c7de66c12..64feef16f 100644 --- a/convex-db/src/test/java/convex/db/sql/ScalingBench.java +++ b/convex-db/src/test/java/convex/db/sql/ScalingBench.java @@ -18,7 +18,7 @@ */ public class ScalingBench { - static final int[] SIZES = {1_000, 10_000, 100_000}; + static final int[] SIZES = {1_000, 10_000, 100_000, 200_000,400_000}; public static void main(String[] args) { System.out.println("=== Convex DB Scaling Benchmark (issue #541) ===\n"); diff --git a/convex-db/src/test/java/convex/db/sql/SecondaryIndexBench.java b/convex-db/src/test/java/convex/db/sql/SecondaryIndexBench.java new file mode 100644 index 000000000..9b5b54ec1 --- /dev/null +++ b/convex-db/src/test/java/convex/db/sql/SecondaryIndexBench.java @@ -0,0 +1,302 @@ +package convex.db.sql; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import convex.core.crypto.AKeyPair; +import convex.core.data.ABlob; +import convex.core.data.ACell; +import convex.core.data.AVector; +import convex.core.data.Index; +import convex.core.data.Strings; +import convex.core.data.Vectors; +import convex.core.data.prim.CVMLong; +import convex.db.calcite.ConvexType; +import convex.db.lattice.SQLDatabase; +import convex.db.lattice.SQLSchema; + +/** + * Benchmark comparing secondary index lookups in Convex DB vs MariaDB. + * + *

Measures two things per engine: + *

    + *
  1. Heap delta — bytes allocated to serve the result set. + * Index lookup should be proportional to match count, not table size.
    2. + *
  2. Speed — repeated lookups timed over {@value #SPEED_REPS} iterations + * after a {@value #SPEED_WARMUP}-iteration warmup, reporting ops/sec and + * median latency.
    3. + *
+ * + *

Benchmark layout ({@value #TABLE_SIZE} rows, {@value #MATCH_COUNT} matching): + *

    + *
  • Convex full scan / index lookup (lattice API)
    • + *
  • MariaDB full scan / index lookup (JDBC)
    • + *
+ * + *

MariaDB connection (overridable via system properties): + * 

+ *   -Dmariadb.host=localhost  (default)
+ *   -Dmariadb.port=3306       (default)
+ *   -Dmariadb.user=tempuser   (default)
+ *   -Dmariadb.pass=123456     (default)
+ *   -Dmariadb.db=temp         (default)
+ *
+ * + *

Run as a plain main() program (not a JUnit test). + */ +public class SecondaryIndexBench { + + static final int TABLE_SIZE = 100_000; + static final int MATCH_COUNT = 100; // rows with status="rare" + static final int SPEED_WARMUP = 50; // discarded warm-up iterations + static final int SPEED_REPS = 200; // timed iterations + + static final String TBL = "sec_idx_bench"; + + // MariaDB connection params + static final String MARIADB_HOST = System.getProperty("mariadb.host", "localhost"); + static final String MARIADB_PORT = System.getProperty("mariadb.port", "3306"); + static final String MARIADB_USER = System.getProperty("mariadb.user", "tempuser"); + static final String MARIADB_PASS = System.getProperty("mariadb.pass", "123456"); + static final String MARIADB_DB = System.getProperty("mariadb.db", "temp"); + static final String MARIADB_URL = + "jdbc:mariadb://" + MARIADB_HOST + ":" + MARIADB_PORT + "/" + MARIADB_DB + + "?useServerPrepStmts=true"; + + public static void main(String[] args) throws Exception { + System.out.println("=== Secondary Index Heap Bench: Convex DB vs MariaDB ==="); + System.out.printf("Table size: %,d rows, matching rows: %d%n%n", TABLE_SIZE, MATCH_COUNT); + + runConvexBench(); + + boolean mariaAvailable = checkMariaDB(); + if (mariaAvailable) { + runMariaDBBench(); + } else { + System.out.println("[MariaDB] Skipped — not available."); + System.out.println(" Start MariaDB and set -Dmariadb.* to enable."); + } + System.out.printf("DONE"); + } + + // ── Convex benchmarks ───────────────────────────────────────────────────── + + static void runConvexBench() throws Exception { + System.out.println("--- Convex DB ---"); + + SQLSchema schema = SQLDatabase.create("bench", AKeyPair.generate()).tables(); + schema.createTable(TBL, + new String[]{"id", "status", "dept", "score"}, + new ConvexType[]{ConvexType.INTEGER, ConvexType.VARCHAR, + ConvexType.VARCHAR, ConvexType.INTEGER}); + + List> rows = new ArrayList<>(TABLE_SIZE); + for (int i = 0; i < TABLE_SIZE; i++) { + String status = (i < MATCH_COUNT) ? "rare" : "common"; + rows.add(Vectors.of(CVMLong.create(i), Strings.create(status), + Strings.create("eng"), CVMLong.create(50 + i % 50))); + } + schema.insertAll(TBL, rows); + rows = null; + + ACell rare = Strings.create("rare"); + + // Full scan (no index) — heap + gc(); + long h0 = usedHeap(); + long scanCount = convexFullScan(schema, 1, rare); + long scanHeap = usedHeap() - h0; + System.out.printf(" Full scan : %,d rows found, heap delta: %+.2f MB%n", + scanCount, mb(scanHeap)); + + // Full scan — speed + for (int i = 0; i < SPEED_WARMUP; i++) convexFullScan(schema, 1, rare); + long[] scanTimes = new long[SPEED_REPS]; + for (int i = 0; i < SPEED_REPS; i++) { + long t = System.nanoTime(); + convexFullScan(schema, 1, rare); + scanTimes[i] = System.nanoTime() - t; + } + printSpeed(" Full scan ", scanTimes); + + // Index lookup — heap + schema.createIndex(TBL, "status"); + + gc(); + long h2 = usedHeap(); + Index> result = + schema.selectByColumn(TBL, "status", rare); + long indexHeap = usedHeap() - h2; + System.out.printf(" Index lookup: %,d rows found, heap delta: %+.2f MB%n", + result.count(), mb(indexHeap)); + + // Index lookup — speed + for (int i = 0; i < SPEED_WARMUP; i++) schema.selectByColumn(TBL, "status", rare); + long[] idxTimes = new long[SPEED_REPS]; + for (int i = 0; i < SPEED_REPS; i++) { + long t = System.nanoTime(); + schema.selectByColumn(TBL, "status", rare); + idxTimes[i] = System.nanoTime() - t; + } + printSpeed(" Index lookup", idxTimes); + + printRatio(scanHeap, indexHeap); + } + + static long convexFullScan(SQLSchema schema, int colIdx, ACell value) { + Index> all = schema.selectAll(TBL); + long count = 0; + for (var e : all.entrySet()) { + AVector row = e.getValue(); + ACell cell = (colIdx < row.count()) ? row.get(colIdx) : null; + if (value.equals(cell)) count++; + } + return count; + } + + // ── MariaDB benchmarks ──────────────────────────────────────────────────── + + static boolean checkMariaDB() { + try { + setupMariaDB(); + return true; + } catch (Exception e) { + System.out.println("[MariaDB] Not available: " + e.getMessage()); + return false; + } + } + + static void setupMariaDB() throws Exception { + String url = MARIADB_URL + "&user=" + MARIADB_USER + "&password=" + MARIADB_PASS; + try (Connection conn = DriverManager.getConnection(url); + Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP TABLE IF EXISTS " + TBL); + stmt.executeUpdate( + "CREATE TABLE " + TBL + " (" + + "id INT PRIMARY KEY, " + + "status VARCHAR(16), " + + "dept VARCHAR(16), " + + "score INT" + + ") ENGINE=InnoDB"); + } + } + + static void runMariaDBBench() throws Exception { + System.out.println("\n--- MariaDB ---"); + + String url = MARIADB_URL + "&user=" + MARIADB_USER + "&password=" + MARIADB_PASS; + + // Insert rows + try (Connection conn = DriverManager.getConnection(url)) { + conn.setAutoCommit(false); + try (PreparedStatement ps = conn.prepareStatement( + "INSERT INTO " + TBL + " VALUES (?, ?, ?, ?)")) { + for (int i = 0; i < TABLE_SIZE; i++) { + String status = (i < MATCH_COUNT) ? "rare" : "common"; + ps.setInt(1, i); + ps.setString(2, status); + ps.setString(3, "eng"); + ps.setInt(4, 50 + i % 50); + ps.addBatch(); + if (i % 1000 == 999) ps.executeBatch(); + } + ps.executeBatch(); + } + conn.commit(); + } + + String scanSql = "SELECT id FROM " + TBL + " WHERE status = 'rare'"; + + try (Connection conn = DriverManager.getConnection(url)) { + // Full scan — heap + gc(); + long h0 = usedHeap(); + long count = mariaCount(conn, scanSql); + long scanHeap = usedHeap() - h0; + System.out.printf(" Full scan : %,d rows found, heap delta: %+.2f MB%n", + count, mb(scanHeap)); + + // Full scan — speed + for (int i = 0; i < SPEED_WARMUP; i++) mariaCount(conn, scanSql); + long[] scanTimes = new long[SPEED_REPS]; + for (int i = 0; i < SPEED_REPS; i++) { + long t = System.nanoTime(); + mariaCount(conn, scanSql); + scanTimes[i] = System.nanoTime() - t; + } + printSpeed(" Full scan ", scanTimes); + + // Add index + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("CREATE INDEX idx_status ON " + TBL + " (status)"); + } + + // Index lookup — heap + gc(); + long h2 = usedHeap(); + long idxCount = mariaCount(conn, scanSql); + long indexHeap = usedHeap() - h2; + System.out.printf(" Index lookup: %,d rows found, heap delta: %+.2f MB%n", + idxCount, mb(indexHeap)); + + // Index lookup — speed + for (int i = 0; i < SPEED_WARMUP; i++) mariaCount(conn, scanSql); + long[] idxTimes = new long[SPEED_REPS]; + for (int i = 0; i < SPEED_REPS; i++) { + long t = System.nanoTime(); + mariaCount(conn, scanSql); + idxTimes[i] = System.nanoTime() - t; + } + printSpeed(" Index lookup", idxTimes); + + printRatio(scanHeap, indexHeap); + } + } + + static long mariaCount(Connection conn, String sql) throws Exception { + long count = 0; + try (Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery(sql)) { + while (rs.next()) count++; + } + return count; + } + + // ── Helpers ─────────────────────────────────────────────────────────────── + + static void printSpeed(String label, long[] nanosPerOp) { + Arrays.sort(nanosPerOp); + long median = nanosPerOp[nanosPerOp.length / 2]; + long p95 = nanosPerOp[(int) (nanosPerOp.length * 0.95)]; + double opsPerSec = 1_000_000_000.0 / median; + System.out.printf("%s speed : median %,.0f µs p95 %,.0f µs (%,.0f ops/sec)%n", + label, median / 1_000.0, p95 / 1_000.0, opsPerSec); + } + + static void printRatio(long scanHeap, long indexHeap) { + System.out.printf(" Ratio scan/index heap: "); + if (indexHeap > 0) { + System.out.printf("%.1fx (expected ~%.0fx for %d/%d rows)%n", + (double) scanHeap / indexHeap, + (double) TABLE_SIZE / MATCH_COUNT, + TABLE_SIZE, MATCH_COUNT); + } else { + System.out.printf("N/A (index heap ≤ 0)%n"); + } + } + + static void gc() { System.gc(); System.gc(); } + + static long usedHeap() { + Runtime rt = Runtime.getRuntime(); + return rt.totalMemory() - rt.freeMemory(); + } + + static double mb(long bytes) { return bytes / 1024.0 / 1024.0; } +} diff --git a/convex-db/src/test/java/convex/db/sql/TemporalDemo.java b/convex-db/src/test/java/convex/db/sql/TemporalDemo.java new file mode 100644 index 000000000..d7f638ecb --- /dev/null +++ b/convex-db/src/test/java/convex/db/sql/TemporalDemo.java @@ -0,0 +1,148 @@ +package convex.db.sql; + +import java.util.List; + +import convex.core.data.ACell; +import convex.core.data.AVector; +import convex.core.data.prim.CVMLong; +import convex.db.lattice.VersionedSQLSchema; +import convex.db.lattice.VersionedSQLTable; + +/** + * Demo: system-versioned table functionality via {@link VersionedSQLSchema}. + * + * Shows insert, update, deduplication, deletion, full history, and + * point-in-time (AS OF) queries — all through the lattice API. + */ +public class TemporalDemo { + + static final String TABLE = "products"; + + public static void main(String[] args) { + System.out.println("=== Convex Temporal Table Demo ===\n"); + + VersionedSQLSchema schema = VersionedSQLSchema.create(); + schema.createTable(TABLE, new String[]{"id", "name", "price", "stock"}); + System.out.println("Table created: " + TABLE + "\n"); + + // 1. Initial insert + long t1 = insert(schema, 1L, "Widget Pro", 29.99, 100); + System.out.println("[t1] Insert id=1 Widget Pro $29.99 stock=100"); + + sleep(1); + + // 2. Price drop + long t2 = insert(schema, 1L, "Widget Pro", 24.99, 100); + System.out.println("[t2] Update id=1 price → $24.99"); + + sleep(1); + + // 3. Duplicate — should be skipped + boolean written = schema.insert(TABLE, buildRow(1L, "Widget Pro", 24.99, 100)); + System.out.println("[t2b] Duplicate insert id=1: written=" + written + " (expected false)"); + + sleep(1); + + // 4. Stock change + long t3 = insert(schema, 1L, "Widget Pro", 24.99, 75); + System.out.println("[t3] Update id=1 stock → 75"); + + sleep(1); + + // 5. Second product + insert(schema, 2L, "Gadget Plus", 59.99, 50); + System.out.println("[--] Insert id=2 Gadget Plus $59.99 stock=50"); + + sleep(1); + + // 6. Delete product 1 + long t4 = delete(schema, 1L); + System.out.println("[t4] Delete id=1\n"); + + // ── Current state ──────────────────────────────────────────────────── + System.out.println("--- Current state ---"); + printCurrentRow(schema, 1L); + printCurrentRow(schema, 2L); + System.out.println(); + + // ── Full history for product 1 ──────────────────────────────────────── + System.out.println("--- Full history: id=1 ---"); + printHistory(schema, 1L); + System.out.println(); + + // ── Point-in-time queries ───────────────────────────────────────────── + System.out.println("--- AS OF t1 (just after initial insert) ---"); + printAsOf(schema, 1L, t1); + + System.out.println("--- AS OF t2 (after price drop) ---"); + printAsOf(schema, 1L, t2); + + System.out.println("--- AS OF t3 (after stock update) ---"); + printAsOf(schema, 1L, t3); + + System.out.println("--- AS OF t4 (after deletion) ---"); + printAsOf(schema, 1L, t4); + + System.out.println("Done."); + } + + // ── Helpers ─────────────────────────────────────────────────────────────── + + static long insert(VersionedSQLSchema schema, long id, Object... rest) { + long ts = System.nanoTime(); + schema.insert(TABLE, buildRow(id, rest)); + return ts; + } + + static long delete(VersionedSQLSchema schema, long id) { + long ts = System.nanoTime(); + schema.deleteByKey(TABLE, CVMLong.create(id)); + return ts; + } + + @SuppressWarnings("unchecked") + static AVector buildRow(long id, Object... rest) { + Object[] all = new Object[1 + rest.length]; + all[0] = id; + System.arraycopy(rest, 0, all, 1, rest.length); + return convex.core.data.Vectors.of(all); + } + + static void printCurrentRow(VersionedSQLSchema schema, long id) { + AVector row = schema.selectByKey(TABLE, CVMLong.create(id)); + System.out.printf(" id=%-4d %s%n", id, row == null ? "(deleted / not found)" : row); + } + + static void printHistory(VersionedSQLSchema schema, long id) { + List> versions = schema.getHistory(TABLE, CVMLong.create(id)); + if (versions.isEmpty()) { System.out.println(" (no history)"); return; } + System.out.printf(" %-8s %-22s %s%n", "Type", "Nanotime", "Values"); + for (AVector entry : versions) { + long ts = ((CVMLong) entry.get(1)).longValue(); + long ct = ((CVMLong) entry.get(2)).longValue(); + AVector vals = VersionedSQLTable.getHistoryValues(entry); + System.out.printf(" %-8s %-22d %s%n", changeType(ct), ts, vals == null ? "DELETED" : vals); + } + } + + static void printAsOf(VersionedSQLSchema schema, long id, long nanotime) { + AVector entry = schema.getAsOf(TABLE, CVMLong.create(id), nanotime); + if (entry == null) { System.out.println(" (no record at this time)\n"); return; } + long ct = ((CVMLong) entry.get(2)).longValue(); + AVector vals = VersionedSQLTable.getHistoryValues(entry); + System.out.printf(" [%s] %s%n%n", changeType(ct), vals == null ? "DELETED" : vals); + } + + static String changeType(long ct) { + return switch ((int) ct) { + case (int) VersionedSQLTable.CT_INSERT -> "INSERT"; + case (int) VersionedSQLTable.CT_UPDATE -> "UPDATE"; + case (int) VersionedSQLTable.CT_DELETE -> "DELETE"; + default -> "UNKNOWN"; + }; + } + + static void sleep(long ms) { + try { Thread.sleep(ms); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } + } +} diff --git a/convex-db/src/test/java/convex/db/store/EtchStoreCompactTest.java b/convex-db/src/test/java/convex/db/store/EtchStoreCompactTest.java new file mode 100644 index 000000000..543e2a963 --- /dev/null +++ b/convex-db/src/test/java/convex/db/store/EtchStoreCompactTest.java @@ -0,0 +1,179 @@ +package convex.db.store; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import convex.core.data.ACell; +import convex.core.data.Blob; +import convex.core.data.Hash; +import convex.core.data.Ref; +import convex.core.data.Vectors; +import convex.core.data.prim.CVMLong; +import convex.etch.EtchStore; + +/** + * Tests for {@link EtchStore#compact(File)}. + * + *

Verifies that: + *

    + *
  • Cells reachable from the root survive compaction
    • + *
  • The compacted file is no larger than the source
    • + *
  • After many writes the compacted file is strictly smaller
    • + *
  • The source file is not modified
    • + *
+ */ +public class EtchStoreCompactTest { + + private EtchStore src; + private EtchStore dst; + private File srcFile; + private File dstFile; + + @AfterEach + void teardown() { + if (src != null) src.close(); + if (dst != null) dst.close(); + if (srcFile != null && srcFile.exists()) srcFile.delete(); + if (dstFile != null && dstFile.exists()) dstFile.delete(); + } + + EtchStore openTemp(String prefix) throws IOException { + File f = File.createTempFile(prefix, ".etch"); + f.deleteOnExit(); + if (prefix.equals("src")) srcFile = f; + else dstFile = f; + return EtchStore.create(f); + } + + /** Returns the data length recorded in the Etch file header. */ + static long etchDataBytes(File file) throws Exception { + try (RandomAccessFile raf = new RandomAccessFile(file, "r")) { + raf.seek(4); + return raf.readLong(); + } + } + + // ── Basic survival ──────────────────────────────────────────────────────── + + @Test + void testRootSurvivesCompaction() throws Exception { + src = openTemp("src"); + dstFile = File.createTempFile("dst", ".etch"); + dstFile.deleteOnExit(); + + ACell root = Blob.wrap(new byte[128]); + src.setRootData(root); + + dst = src.compact(dstFile); + + Ref found = dst.refForHash(Hash.get(root)); + assertNotNull(found); + assertEquals(root, found.getValue()); + } + + @Test + void testNestedCellsSurviveCompaction() throws Exception { + src = openTemp("src"); + dstFile = File.createTempFile("dst", ".etch"); + dstFile.deleteOnExit(); + + // Use blobs larger than MAX_EMBEDDED_LENGTH (140 bytes) so they are stored + // as separate non-embedded cells rather than inlined in the vector encoding. + ACell b1 = Blob.wrap(new byte[200]); + ACell b2 = Blob.wrap(new byte[200]); + ACell root = Vectors.of(b1, b2); + src.setRootData(root); + + dst = src.compact(dstFile); + + assertNotNull(dst.refForHash(Hash.get(root))); + assertNotNull(dst.refForHash(Hash.get(b1))); + assertNotNull(dst.refForHash(Hash.get(b2))); + } + + @Test + void testRootHashPreserved() throws Exception { + src = openTemp("src"); + dstFile = File.createTempFile("dst", ".etch"); + dstFile.deleteOnExit(); + + ACell root = CVMLong.create(42L); + src.setRootData(root); + + dst = src.compact(dstFile); + + ACell dstRoot = dst.getRootData(); + assertEquals(root, dstRoot); + } + + // ── File size ───────────────────────────────────────────────────────────── + + @Test + void testCompactedFileNoLargerThanSource() throws Exception { + src = openTemp("src"); + dstFile = File.createTempFile("dst", ".etch"); + dstFile.deleteOnExit(); + + ACell root = Blob.wrap(new byte[256]); + src.setRootData(root); + + long srcBytes = etchDataBytes(srcFile); + dst = src.compact(dstFile); + dst.close(); dst = null; + long dstBytes = etchDataBytes(dstFile); + + assertTrue(dstBytes <= srcBytes, + "Compacted file (" + dstBytes + ") should be <= source (" + srcBytes + ")"); + } + + @Test + void testCompactionReducesSizeAfterManyWrites() throws Exception { + src = openTemp("src"); + dstFile = File.createTempFile("dst", ".etch"); + dstFile.deleteOnExit(); + + // Write many different roots to accumulate dead cells + for (int i = 0; i < 100; i++) { + src.setRootData(Blob.wrap(new byte[256])); + } + // Final root is a distinct value + ACell finalRoot = Vectors.of(CVMLong.create(999L)); + src.setRootData(finalRoot); + src.flush(); + + long srcBytes = etchDataBytes(srcFile); + dst = src.compact(dstFile); + dst.close(); dst = null; + long dstBytes = etchDataBytes(dstFile); + + assertTrue(dstBytes < srcBytes, + "Compacted file (" + dstBytes + ") should be smaller than bloated source (" + srcBytes + ")"); + assertEquals(finalRoot, EtchStore.create(dstFile).getRootData()); + } + + // ── Source unchanged ────────────────────────────────────────────────────── + + @Test + void testSourceFileUnchangedAfterCompaction() throws Exception { + src = openTemp("src"); + dstFile = File.createTempFile("dst", ".etch"); + dstFile.deleteOnExit(); + + ACell root = Blob.wrap(new byte[128]); + src.setRootData(root); + long srcBytesBefore = etchDataBytes(srcFile); + + dst = src.compact(dstFile); + + long srcBytesAfter = etchDataBytes(srcFile); + assertEquals(srcBytesBefore, srcBytesAfter, "Source file must not be modified during compaction"); + } +} diff --git a/convex-db/src/test/java/convex/db/store/SegmentedEtchStoreTest.java b/convex-db/src/test/java/convex/db/store/SegmentedEtchStoreTest.java new file mode 100644 index 000000000..c7219598e --- /dev/null +++ b/convex-db/src/test/java/convex/db/store/SegmentedEtchStoreTest.java @@ -0,0 +1,216 @@ +package convex.db.store; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import convex.core.data.ACell; +import convex.core.data.Blob; +import convex.core.data.Hash; +import convex.core.data.Ref; + +/** + * Tests for SegmentedEtchStore: write/read across segments, seal rotation, + * compaction status tracking, and online sealed-segment compaction. + * + *

Tests use Blob values (non-embedded) so cells are actually written to the + * Etch file. Each test calls setRootData to establish a root, which is required + * for root-based compaction to traverse the reachable cell tree. + */ +public class SegmentedEtchStoreTest { + + private File dir; + private SegmentedEtchStore store; + + @BeforeEach + void setup() throws IOException { + dir = Files.createTempDirectory("segmented-etch-test-").toFile(); + store = SegmentedEtchStore.createFresh(dir); + } + + @AfterEach + void teardown() { + if (store != null) store.close(); + File[] files = dir.listFiles(); + if (files != null) for (File f : files) f.delete(); + dir.delete(); + } + + /** Creates a non-embedded Blob of given size, suitable for Etch persistence. */ + static Blob makeBlob(int size, byte fill) { + byte[] bs = new byte[size]; + for (int i = 0; i < size; i++) bs[i] = fill; + return Blob.wrap(bs); + } + + // ── Basic write/read ────────────────────────────────────────────────────── + + @Test + void testWriteAndReadFromHot() throws IOException { + ACell val = makeBlob(64, (byte) 1); + store.setRootData(val); + + Ref found = store.refForHash(Hash.get(val)); + assertNotNull(found); + assertEquals(val, found.getValue()); + } + + @Test + void testMissingReturnsNull() throws IOException { + ACell other = makeBlob(64, (byte) 99); + assertNull(store.refForHash(Hash.get(other))); + } + + // ── Seal rotation ───────────────────────────────────────────────────────── + + @Test + void testSealCreatesWarmFile() throws IOException { + store.setRootData(makeBlob(64, (byte) 1)); + assertEquals(0, store.getSealedCount()); + store.sealHot(); + assertEquals(1, store.getSealedCount()); + assertTrue(store.getHotFile().exists()); + } + + @Test + void testReadFromSealedAfterRotation() throws IOException { + ACell val = makeBlob(64, (byte) 7); + store.setRootData(val); + store.sealHot(); // val is now in a sealed segment + + Ref found = store.refForHash(Hash.get(val)); + assertNotNull(found, "Should still find value after sealing"); + assertEquals(val, found.getValue()); + } + + @Test + void testMultipleSealedSegments() throws IOException { + ACell v1 = makeBlob(64, (byte) 1); + ACell v2 = makeBlob(64, (byte) 2); + ACell v3 = makeBlob(64, (byte) 3); + + store.setRootData(v1); + store.sealHot(); + store.setRootData(v2); + store.sealHot(); + store.setRootData(v3); // in hot + + assertEquals(2, store.getSealedCount()); + assertNotNull(store.refForHash(Hash.get(v1)), "v1 in oldest sealed"); + assertNotNull(store.refForHash(Hash.get(v2)), "v2 in newest sealed"); + assertNotNull(store.refForHash(Hash.get(v3)), "v3 in hot"); + } + + // ── Compaction status ───────────────────────────────────────────────────── + + @Test + void testNeedsCompactionAfterSeal() throws IOException { + store.setRootData(makeBlob(64, (byte) 1)); + store.sealHot(); + + assertTrue(store.needsCompaction(0), "Fresh sealed segment needs compaction"); + assertFalse(store.isCompacted(0)); + } + + @Test + void testIsCompactedAfterCompaction() throws IOException { + store.setRootData(makeBlob(64, (byte) 1)); + store.sealHot(); + store.compactSealed(0); + + assertTrue(store.isCompacted(0)); + assertFalse(store.needsCompaction(0)); + } + + @Test + void testCompactedFilenameHasSuffix() throws IOException { + store.setRootData(makeBlob(64, (byte) 1)); + store.sealHot(); + store.compactSealed(0); + + File compactedFile = store.getSealed().get(0).getFile(); + assertTrue(compactedFile.getName().endsWith("-c.etch"), + "Compacted file should have -c.etch suffix, got: " + compactedFile.getName()); + } + + @Test + void testOriginalFileDeletedAfterCompaction() throws IOException { + store.setRootData(makeBlob(64, (byte) 1)); + store.sealHot(); + + File originalFile = store.getSealed().get(0).getFile(); + store.compactSealed(0); + + assertFalse(originalFile.exists(), "Original uncompacted file should be deleted"); + } + + // ── Data integrity after compaction ─────────────────────────────────────── + + @Test + void testReadFromCompactedSegment() throws IOException { + ACell val = makeBlob(64, (byte) 42); + store.setRootData(val); + store.sealHot(); + store.compactSealed(0); + + Ref found = store.refForHash(Hash.get(val)); + assertNotNull(found, "Value should survive compaction"); + assertEquals(val, found.getValue()); + } + + @Test + void testCompactAllSealed() throws IOException { + ACell v1 = makeBlob(64, (byte) 10); + ACell v2 = makeBlob(64, (byte) 20); + + store.setRootData(v1); + store.sealHot(); + store.setRootData(v2); + store.sealHot(); + + store.compactAllSealed(); + + assertTrue(store.isCompacted(0)); + assertTrue(store.isCompacted(1)); + assertNotNull(store.refForHash(Hash.get(v1))); + assertNotNull(store.refForHash(Hash.get(v2))); + } + + @Test + void testCompactSealedIsIdempotent() throws IOException { + store.setRootData(makeBlob(64, (byte) 1)); + store.sealHot(); + store.compactSealed(0); + // Second call should be a no-op — no exception, still readable + store.compactSealed(0); + + assertTrue(store.isCompacted(0)); + } + + // ── Reopen ──────────────────────────────────────────────────────────────── + + @Test + void testReopenRecognisesCompactedSegments() throws IOException { + ACell val = makeBlob(64, (byte) 55); + store.setRootData(val); + store.sealHot(); + store.compactSealed(0); + store.close(); + + // Reopen — compacted segment (-c.etch) should be picked up + store = SegmentedEtchStore.open(dir); + assertEquals(1, store.getSealedCount()); + assertTrue(store.isCompacted(0)); + assertNotNull(store.refForHash(Hash.get(val))); + } +}