Performance enhancements, time indices and secondary indices for Convex-db

convex-core/src/main/java/convex/core/store/ACachedStore.java

@@ -13,7 +13,7 @@
  */
 public abstract class ACachedStore extends AStore {
 
-	protected final RefCache refCache=RefCache.create(10000);
+	protected final RefCache refCache=RefCache.create(Integer.getInteger("convex.cache.size", 10000));
 
 	/**
 	 * Store-bound encoder. Manages thread-local store context during decode.

convex-core/src/main/java/convex/etch/Etch.java

@@ -292,14 +292,9 @@ private synchronized MappedByteBuffer createBuffer(int regionIndex) throws IOExc
 		long pos=((long)regionIndex)*MAX_REGION_SIZE;
 
 		// Expand region size until big enough for current database plus appropriate margin
-		int length;
-		if (regionIndex==0) {
-			length=1<<16;
-			while((length<MAX_REGION_SIZE)&&((pos+length)<(dataLength+REGION_MARGIN))) {
-				length*=2;
-			}
-		} else {
-			length=(int)MAX_REGION_SIZE;
+		int length=1<<16;
+		while((length<MAX_REGION_SIZE)&&((pos+length)<(dataLength+REGION_MARGIN))) {
+			length*=2;
 		}
 
 		length+=REGION_MARGIN; // include margin in buffer length

convex-core/src/main/java/convex/etch/EtchStore.java

@@ -287,9 +287,37 @@ public <T extends ACell> Ref<T> setRootData(T data) throws IOException {
 		return ref;
 	}
 
+	/**
+	 * Compacts this store into a new Etch file containing only the cells reachable
+	 * from the current root. Dead cells accumulated through updates are discarded.
+	 *
+	 * <p>The source store must have its root data set (via {@link #setRootData}).
+	 * The target file must not exist or be empty.
+	 *
+	 * <p>Compaction ratio depends on write history: a store that has had many updates
+	 * to the same rows may see 10× or more reduction because every Index node touched
+	 * during an insert is written as a new cell in the append-only file.
+	 *
+	 * <p>Thread safety: the caller is responsible for ensuring no concurrent writes
+	 * to this store during compaction.
+	 *
+	 * @param targetFile Destination file for the compacted store (must be writable)
+	 * @return New EtchStore backed by the compacted file
+	 * @throws IOException If an IO error occurs during read or write
+	 */
+	public EtchStore compact(File targetFile) throws IOException {
+		Ref<?> rootRef = getRootRef();
+		if (rootRef == null) throw new IOException("Source store has no root data — call setRootData first");
+		// Load the full reachable cell tree into memory/cache in the source store context
+		rootRef = storeTopRef(rootRef, Ref.PERSISTED, null);
+		EtchStore target = EtchStore.create(targetFile);
+		target.setRootData(rootRef.getValue());
+		return target;
+	}
+
 	/**
 	 * Gets the underlying Etch instance
-	 * 
+	 *
 	 * @return Etch instance
 	 */
 	public Etch getEtch() {

convex-db/pom.xml

@@ -90,5 +90,13 @@
 			<version>42.7.7</version>
 			<scope>test</scope>
 		</dependency>
+
+		<!-- MariaDB JDBC driver for comparison benchmarks -->
+		<dependency>
+			<groupId>org.mariadb.jdbc</groupId>
+			<artifactId>mariadb-java-client</artifactId>
+			<version>3.4.1</version>
+			<scope>test</scope>
+		</dependency>
 	</dependencies>
 </project>

convex-db/src/main/java/convex/db/calcite/ConvexSchema.java

@@ -2,6 +2,7 @@
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.calcite.schema.Schema;
 import org.apache.calcite.schema.Table;
@@ -27,6 +28,12 @@ public class ConvexSchema extends AbstractSchema {
 	private final SQLSchema tables;
 	private final String name;
 
+	/**
+	 * Registry of SQL index names to (tableName, columnName) pairs.
+	 * Populated by CREATE INDEX DDL; used to resolve DROP INDEX by name.
+	 */
+	final ConcurrentHashMap<String, String[]> indexRegistry = new ConcurrentHashMap<>();
+
 	/**
 	 * Creates a new ConvexSchema backed by the given database.
 	 *
@@ -130,4 +137,43 @@ public boolean tableExists(String tableName) {
 	public ConvexTable getConvexTable(String tableName) {
 		return new ConvexTable(this, tableName);
 	}
+
+	/**
+	 * Creates a secondary index via DDL.
+	 *
+	 * @param indexName  SQL index name (for later DROP INDEX lookup)
+	 * @param tableName  Table to index
+	 * @param columnName Column to index
+	 * @param ifNotExists if true, silently succeed if already exists
+	 * @return true if index was created
+	 */
+	public boolean createIndex(String indexName, String tableName, String columnName,
+			boolean ifNotExists) {
+		if (tables.hasIndex(tableName, columnName)) {
+			if (ifNotExists) return false;
+			throw new IllegalStateException(
+					"Index on " + tableName + "(" + columnName + ") already exists");
+		}
+		boolean created = tables.createIndex(tableName, columnName);
+		if (created) {
+			indexRegistry.put(indexName.toUpperCase(), new String[]{tableName, columnName});
+		}
+		return created;
+	}
+
+	/**
+	 * Drops a secondary index by its SQL name.
+	 *
+	 * @param indexName SQL index name
+	 * @param ifExists  if true, silently succeed if not found
+	 * @return true if index was dropped
+	 */
+	public boolean dropIndex(String indexName, boolean ifExists) {
+		String[] pair = indexRegistry.remove(indexName.toUpperCase());
+		if (pair == null) {
+			if (ifExists) return false;
+			throw new IllegalStateException("Index not found: " + indexName);
+		}
+		return tables.dropIndex(pair[0], pair[1]);
+	}
 }

convex-db/src/main/java/convex/db/calcite/ConvexTable.java

@@ -47,6 +47,12 @@ public class ConvexTable extends AbstractQueryableTable
 	private final ConvexSchema schema;
 	private final String tableName;
 
+	// Lazily cached column metadata — column names and types are stable for a
+	// given table definition and expensive to re-derive from the lattice on every
+	// getRowType() / getStatistic() call during query planning.
+	private volatile String[] cachedColumnNames;
+	private volatile ConvexColumnType[] cachedColumnTypes;
+
 	public ConvexTable(ConvexSchema schema, String tableName) {
 		super(Object[].class);
 		this.schema = schema;
@@ -57,9 +63,8 @@ public ConvexTable(ConvexSchema schema, String tableName) {
 	public RelDataType getRowType(RelDataTypeFactory typeFactory) {
 		RelDataTypeFactory.Builder builder = typeFactory.builder();
 
-		SQLSchema tables = schema.getTables();
-		String[] columnNames = tables.getColumnNames(tableName);
-		ConvexColumnType[] columnTypes = tables.getColumnTypes(tableName);
+		String[] columnNames = getColumnNames();
+		ConvexColumnType[] columnTypes = getColumnTypes();
 
 		if (columnNames != null && columnTypes != null) {
 			for (int i = 0; i < columnNames.length; i++) {
@@ -71,6 +76,13 @@ public RelDataType getRowType(RelDataTypeFactory typeFactory) {
 		return builder.build();
 	}
 
+	private String[] getColumnNames() {
+		if (cachedColumnNames == null) {
+			cachedColumnNames = schema.getTables().getColumnNames(tableName);
+		}
+		return cachedColumnNames;
+	}
+
 	/**
 	 * Returns a ConvexTableScan in CONVEX convention. This is Calcite's
 	 * standard extension point for custom adapters — bypasses EnumerableTableScan
@@ -87,7 +99,10 @@ public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable relOptTable)
 	}
 
 	public ConvexColumnType[] getColumnTypes() {
-		return schema.getTables().getColumnTypes(tableName);
+		if (cachedColumnTypes == null) {
+			cachedColumnTypes = schema.getTables().getColumnTypes(tableName);
+		}
+		return cachedColumnTypes;
 	}
 
 	/**

convex-db/src/main/java/convex/db/calcite/ConvexTableEnumerator.java

@@ -1,64 +1,122 @@
 package convex.db.calcite;
 
-import java.util.ArrayList;
-import java.util.List;
+import java.util.Iterator;
 
 import convex.core.data.ABlob;
 import convex.core.data.ACell;
 import convex.core.data.AVector;
 import convex.core.data.Index;
+import convex.db.lattice.RowBlock;
 import convex.db.lattice.SQLRow;
 import convex.db.lattice.SQLSchema;
 import convex.db.lattice.SQLTable;
 
 /**
- * Enumerator that performs a full table scan from a lattice Index.
+ * Enumerator that performs a lazy full table scan over a block-packed row index.
  *
- * <p>Rows are returned in Index key order (sorted by PK).
- * Pre-collects via forEach for O(n) traversal instead of O(n*depth) entryAt().
+ * <p>Iterates the outer block Index one block at a time, decoding each block's
+ * bytes once into a row buffer, then draining that buffer row by row.
+ *
+ * <p>Two traversal paths:
+ * <ul>
+ *   <li><b>blockVec path</b> — when slot 4 of the table state holds an
+ *       {@code AVector<ACell>} of live block blobs. Uses
+ *       {@link AVector#iterator()} for O(n)-total traversal (one VectorTree
+ *       descent rather than one per element).</li>
+ *   <li><b>Index path</b> — sequential {@code entryAt(i)} traversal of the
+ *       row block Index; also O(n) total via the trie's sequential iterator.</li>
+ * </ul>
  */
 public class ConvexTableEnumerator extends ConvexEnumerator {
 
-	private final List<ACell[]> rows;
-	private int position = -1;
+	/** Non-null when blockVec path is active; stored to allow reset(). */
+	private final AVector<ACell> blockVec;
+	/** O(n)-total iterator over blockVec; reset()-able by re-creating from blockVec. */
+	private Iterator<ACell> blockVecIter;
+	/** Fallback Index path (used when blockVec == null). */
+	private final Index<ABlob, ACell> blockIndex;
+	private final int colCount;
+	private final long blockCount;
+
+	private long blockPos = -1;
+	@SuppressWarnings("unchecked")
+	private AVector<ACell>[] blockRowBuf = new AVector[0];
+	private int blockRowBufSize = 0;
+	private int rowPos = 0;
 
-	/**
-	 * Creates an enumerator for a full table scan.
-	 */
 	public ConvexTableEnumerator(SQLSchema tables, String tableName) {
 		SQLTable table = tables.getLiveTable(tableName);
 		if (table == null) {
-			this.rows = List.of();
+			this.blockVec = null;
+			this.blockVecIter = null;
+			this.blockIndex = Index.none();
+			this.colCount = 0;
+			this.blockCount = 0;
 			return;
 		}
-		Index<ABlob, AVector<ACell>> rawRows = table.getRows();
-		if (rawRows == null) {
-			this.rows = List.of();
-			return;
+		AVector<ACell> bv = table.getBlockVec();
+		if (bv != null) {
+			// blockVec path: iterator is O(n) total (visits each VectorTree node once)
+			this.blockVec = bv;
+			this.blockVecIter = bv.iterator();
+			this.blockIndex = null;
+			this.blockCount = bv.count();
+		} else {
+			this.blockVec = null;
+			this.blockVecIter = null;
+			Index<ABlob, ACell> rows = table.getRows();
+			this.blockIndex = rows != null ? rows : Index.none();
+			this.blockCount = this.blockIndex.count();
 		}
-		// Single-pass tree traversal, skip tombstones, unwrap values
-		List<ACell[]> collected = new ArrayList<>((int) Math.min(rawRows.count(), Integer.MAX_VALUE));
-		rawRows.forEach((k, v) -> {
-			if (SQLRow.isLive(v)) {
-				collected.add(SQLRow.getValues(v).toCellArray());
-			}
-		});
-		this.rows = collected;
+		AVector<AVector<ACell>> schema = table.getSchema();
+		this.colCount = schema != null ? (int) schema.count() : 0;
 	}
 
 	@Override
 	public boolean moveNext() {
-		if (++position < rows.size()) {
-			currentRow = rows.get(position);
-			return true;
+		while (true) {
+			// Drain rows buffered from the current block
+			while (rowPos < blockRowBufSize) {
+				AVector<ACell> row = blockRowBuf[rowPos++];
+				if (!SQLRow.isLive(row)) continue;
+				ACell[] arr = SQLRow.getValues(row).toCellArray();
+				if (arr.length < colCount) {
+					ACell[] padded = new ACell[colCount];
+					System.arraycopy(arr, 0, padded, 0, arr.length);
+					arr = padded;
+				}
+				currentRow = arr;
+				return true;
+			}
+			// Advance to next block — decode it once into blockRowBuf
+			blockPos++;
+			if (blockPos >= blockCount) {
+				currentRow = null;
+				return false;
+			}
+			ACell block = (blockVecIter != null)
+					? blockVecIter.next()
+					: blockIndex.entryAt(blockPos).getValue();
+			int n = RowBlock.isBlock(block) ? RowBlock.count(block) : 0;
+			if (n > blockRowBuf.length) {
+				@SuppressWarnings("unchecked")
+				AVector<ACell>[] newBuf = new AVector[n];
+				blockRowBuf = newBuf;
+			}
+			blockRowBufSize = 0;
+			rowPos = 0;
+			final int[] idx = {0};
+			RowBlock.forEach(block, (pk, row) -> blockRowBuf[idx[0]++] = row);
+			blockRowBufSize = idx[0];
 		}
-		currentRow = null;
-		return false;
 	}
 
 	@Override
 	public void reset() {
-		position = -1;
+		blockPos = -1;
+		blockRowBufSize = 0;
+		rowPos = 0;
 		currentRow = null;
+		if (blockVec != null) blockVecIter = blockVec.iterator();
 	}
 }