feat(schema): ClickHouse data-skipping indexes and engine SETTINGS

lohanidamodar · lohanidamodar · commit 18a77b0e315b · 2026-04-28T09:48:12.000Z
The ClickHouse schema previously hard-coded `TYPE minmax GRANULARITY 3`
for every index and had no way to emit a `SETTINGS k=v` clause. Both are
needed by real ClickHouse-backed services that pick algorithms per
column and tune engine settings (e.g. `index_granularity`,
`allow_nullable_key`).

This change adds:

- `Schema\ClickHouse\SkipIndexAlgorithm` enum — `MinMax`, `Set`,
  `BloomFilter`, `NgramBloomFilter`, `TokenBloomFilter`, `Inverted`.
- `Schema\ClickHouse\SkipIndex` value object carrying the algorithm,
  granularity, and algorithm-specific args (e.g. `[100]` for `set(100)`,
  `[4, 1024, 3, 0]` for `ngrambf_v1(...)`).
- `Table::dataSkippingIndex()` — attach a skip index to the blueprint.
  Default granularity 1, default no algorithm args.
- `Table::settings()` — set table-level engine SETTINGS, with a
  conservative key/value allow-list to keep DDL safe.
- `Schema\ClickHouse::create()` renders both. SETTINGS is emitted after
  TTL. The existing minmax compile path is unchanged for backward
  compatibility — callers using `index()` keep their current output.

Other dialects ignore the new fields, matching how `engine()` and `ttl()`
are already handled.

Tests cover each skip algorithm shape, composite-column skip indexes,
SETTINGS rendering with and without TTL, and validation of bad
granularity, empty columns, bad setting names, and unsafe string values.
diff --git a/README.md b/README.md
@@ -2081,6 +2081,49 @@ $schema->create('events', function (Table $table) {
 
 TTL expressions are emitted verbatim; they must not be empty or contain semicolons. Dialects other than ClickHouse throw `UnsupportedException`.
 
+**Data-skipping indexes** — accelerate WHERE pruning by letting ClickHouse skip whole granules:
+
+```php
+use Utopia\Query\Schema\ClickHouse\SkipIndexAlgorithm;
+
+$schema->create('events', function (Table $table) {
+    $table->bigInteger('id')->primary();
+    $table->string('user_id');
+    $table->string('country');
+    $table->string('text');
+
+    // Default granularity = 1, no algorithm args
+    $table->dataSkippingIndex(['user_id'], SkipIndexAlgorithm::BloomFilter);
+
+    // Set(N) — small fixed value sets
+    $table->dataSkippingIndex(['country'], SkipIndexAlgorithm::Set, granularity: 4, algorithmArgs: [100]);
+
+    // NgramBloomFilter(n, size_bytes, hashes, seed) — text search on `LIKE` / `match`
+    $table->dataSkippingIndex(['text'], SkipIndexAlgorithm::NgramBloomFilter, algorithmArgs: [4, 1024, 3, 0]);
+});
+
+// CREATE TABLE `events` (..., INDEX `skip_user_id` `user_id` TYPE bloom_filter GRANULARITY 1, ...)
+```
+
+The 6 algorithms are `MinMax`, `Set`, `BloomFilter`, `NgramBloomFilter`, `TokenBloomFilter`, `Inverted`. Algorithm-specific arguments are passed via `algorithmArgs` and rendered verbatim — supply them from trusted (developer-controlled) source. Other dialects ignore the call.
+
+**Engine SETTINGS** — emit `SETTINGS k=v` after the TTL clause:
+
+```php
+$schema->create('events', function (Table $table) {
+    $table->bigInteger('id')->primary();
+    $table->settings([
+        'index_granularity' => 8192,
+        'allow_nullable_key' => true, // booleans become 1/0
+    ]);
+});
+
+// CREATE TABLE `events` (...) ENGINE = MergeTree() ORDER BY (`id`)
+//   SETTINGS index_granularity = 8192, allow_nullable_key = 1
+```
+
+Setting names must match `[A-Za-z_][A-Za-z0-9_]*`; string values are restricted to `[A-Za-z0-9_.\-+/]*`. Use ints / floats / booleans for everything else. Other dialects ignore the call.
+
 ### SQLite Schema
 
 ```php
diff --git a/src/Query/Schema/ClickHouse.php b/src/Query/Schema/ClickHouse.php
@@ -9,6 +9,7 @@
 use Utopia\Query\QuotesIdentifiers;
 use Utopia\Query\Schema;
 use Utopia\Query\Schema\ClickHouse\Engine;
+use Utopia\Query\Schema\ClickHouse\SkipIndex;
 use Utopia\Query\Schema\Feature\ColumnComments;
 use Utopia\Query\Schema\Feature\DropPartition;
 use Utopia\Query\Schema\Feature\TableComments;
@@ -183,6 +184,14 @@ public function create(string $table, callable $definition, bool $ifNotExists =
                 . ' ' . $expr . ' TYPE minmax GRANULARITY 3';
         }
 
+        foreach ($blueprint->skipIndexes as $skip) {
+            $cols = \array_map(fn (string $c): string => $this->quote($c), $skip->columns);
+            $expr = \count($cols) === 1 ? $cols[0] : '(' . \implode(', ', $cols) . ')';
+            $columnDefs[] = 'INDEX ' . $this->quote($skip->name)
+                . ' ' . $expr . ' TYPE ' . $this->compileSkipAlgorithm($skip)
+                . ' GRANULARITY ' . $skip->granularity;
+        }
+
         if (! empty($blueprint->foreignKeys)) {
             throw new UnsupportedException('Foreign keys are not supported in ClickHouse.');
         }
@@ -211,9 +220,40 @@ public function create(string $table, callable $definition, bool $ifNotExists =
             $sql .= ' TTL ' . $blueprint->ttl;
         }
 
+        if (! empty($blueprint->settings)) {
+            $kv = [];
+            foreach ($blueprint->settings as $k => $v) {
+                $kv[] = $k . ' = ' . $v;
+            }
+            $sql .= ' SETTINGS ' . \implode(', ', $kv);
+        }
+
         return new Statement($sql, [], executor: $this->executor);
     }
 
+    /**
+     * Render a `TYPE <algorithm>(args)` fragment for a data-skipping index.
+     *
+     * String args are emitted as single-quoted SQL literals (with `'` doubled);
+     * numeric args are emitted verbatim. Argument values come from the
+     * application — never from untrusted input.
+     */
+    private function compileSkipAlgorithm(SkipIndex $skip): string
+    {
+        if ($skip->algorithmArgs === []) {
+            return $skip->algorithm->value;
+        }
+
+        $args = \array_map(
+            fn (string|int|float $arg): string => \is_string($arg)
+                ? "'" . \str_replace("'", "''", $arg) . "'"
+                : (string) $arg,
+            $skip->algorithmArgs,
+        );
+
+        return $skip->algorithm->value . '(' . \implode(', ', $args) . ')';
+    }
+
     /**
      * Compile an engine declaration: `<Name>` or `<Name>(<args...>)`.
      *
diff --git a/src/Query/Schema/ClickHouse/SkipIndex.php b/src/Query/Schema/ClickHouse/SkipIndex.php
@@ -0,0 +1,33 @@
+<?php
+
+namespace Utopia\Query\Schema\ClickHouse;
+
+use Utopia\Query\Exception\ValidationException;
+
+readonly class SkipIndex
+{
+    /**
+     * @param  list<string>  $columns
+     * @param  list<string|int|float>  $algorithmArgs  Args for parameterized algorithms
+     *                                                  (e.g. [3] for set(3),
+     *                                                  [0.01] for bloom_filter(0.01),
+     *                                                  [4, 1024, 3, 0] for ngrambf_v1(n, size_bytes, hashes, seed))
+     */
+    public function __construct(
+        public string $name,
+        public array $columns,
+        public SkipIndexAlgorithm $algorithm,
+        public array $algorithmArgs = [],
+        public int $granularity = 1,
+    ) {
+        if (! \preg_match('/^[A-Za-z_][A-Za-z0-9_]*$/', $name)) {
+            throw new ValidationException('Invalid skip index name: ' . $name);
+        }
+        if ($columns === []) {
+            throw new ValidationException('Skip index requires at least one column.');
+        }
+        if ($granularity < 1) {
+            throw new ValidationException('Skip index granularity must be >= 1.');
+        }
+    }
+}
diff --git a/src/Query/Schema/ClickHouse/SkipIndexAlgorithm.php b/src/Query/Schema/ClickHouse/SkipIndexAlgorithm.php
@@ -0,0 +1,13 @@
+<?php
+
+namespace Utopia\Query\Schema\ClickHouse;
+
+enum SkipIndexAlgorithm: string
+{
+    case MinMax = 'minmax';
+    case Set = 'set';
+    case BloomFilter = 'bloom_filter';
+    case NgramBloomFilter = 'ngrambf_v1';
+    case TokenBloomFilter = 'tokenbf_v1';
+    case Inverted = 'inverted';
+}
diff --git a/src/Query/Schema/Table.php b/src/Query/Schema/Table.php
@@ -4,6 +4,8 @@
 
 use Utopia\Query\Exception\ValidationException;
 use Utopia\Query\Schema\ClickHouse\Engine;
+use Utopia\Query\Schema\ClickHouse\SkipIndex;
+use Utopia\Query\Schema\ClickHouse\SkipIndexAlgorithm;
 
 class Table
 {
@@ -51,6 +53,12 @@ class Table
 
     public private(set) ?string $ttl = null;
 
+    /** @var list<SkipIndex> ClickHouse data-skipping indexes (other dialects ignore) */
+    public private(set) array $skipIndexes = [];
+
+    /** @var array<string, string> Table-level engine SETTINGS (ClickHouse only) */
+    public private(set) array $settings = [];
+
     /**
      * Add a table-level CHECK constraint.
      *
@@ -543,4 +551,87 @@ public function ttl(string $expression): static
 
         return $this;
     }
+
+    /**
+     * Attach a ClickHouse data-skipping index. Other dialects ignore this.
+     *
+     * Skip indexes accelerate WHERE clauses by letting ClickHouse skip whole
+     * granules during scanning. Choose the algorithm that matches the column
+     * cardinality and predicate type:
+     *
+     * - `MinMax` — numeric ranges, low cardinality
+     * - `Set(N)` — small fixed value sets (N is the set size cap)
+     * - `BloomFilter(p)` — high cardinality string columns with `=` / `IN`
+     *   predicates (p is the false-positive probability, e.g. 0.01)
+     * - `NgramBloomFilter(n, size, hashes, seed)` — `LIKE` / `match` on text
+     * - `TokenBloomFilter(size, hashes, seed)` — token-style search
+     * - `Inverted` — `LIKE`, `match`, `hasToken` (experimental)
+     *
+     * @param  list<string>  $columns
+     * @param  list<string|int|float>  $algorithmArgs  Algorithm-specific arguments
+     *
+     * @throws ValidationException if the index name or columns are invalid.
+     */
+    public function dataSkippingIndex(
+        array $columns,
+        SkipIndexAlgorithm $algorithm,
+        int $granularity = 1,
+        array $algorithmArgs = [],
+        string $name = '',
+    ): static {
+        if ($name === '') {
+            $name = 'skip_' . \implode('_', $columns);
+        }
+
+        $this->skipIndexes[] = new SkipIndex($name, $columns, $algorithm, $algorithmArgs, $granularity);
+
+        return $this;
+    }
+
+    /**
+     * Set table-level engine SETTINGS (ClickHouse only). Other dialects ignore.
+     *
+     * Compiled as `SETTINGS k=v, ...` after the TTL clause. Booleans become
+     * `1` / `0`, ints/floats are stringified, strings are passed through after
+     * a conservative character allow-list check.
+     *
+     * Calling this method replaces previously-set settings.
+     *
+     * @param  array<string, string|int|float|bool>  $settings
+     *
+     * @throws ValidationException if any key is not a valid identifier or any
+     *                             string value contains characters outside the
+     *                             allow-list.
+     */
+    public function settings(array $settings): static
+    {
+        $sanitized = [];
+
+        foreach ($settings as $key => $value) {
+            if (! \preg_match('/^[A-Za-z_][A-Za-z0-9_]*$/', $key)) {
+                throw new ValidationException('Invalid setting name: ' . $key);
+            }
+
+            if (\is_bool($value)) {
+                $sanitized[$key] = $value ? '1' : '0';
+            } elseif (\is_int($value) || \is_float($value)) {
+                $sanitized[$key] = (string) $value;
+            } elseif (\is_string($value)) {
+                if (! \preg_match('/^[A-Za-z0-9_.\-+\/]*$/', $value)) {
+                    throw new ValidationException(
+                        'Invalid setting value for ' . $key . ': must match [A-Za-z0-9_.\\-+/]*'
+                    );
+                }
+                $sanitized[$key] = $value;
+            } else {
+                throw new ValidationException(
+                    'Setting value for ' . $key . ' must be string, int, float, or bool.'
+                );
+            }
+        }
+
+        $this->settings = $sanitized;
+
+        return $this;
+    }
 }
diff --git a/tests/Query/Schema/ClickHouseTest.php b/tests/Query/Schema/ClickHouseTest.php
