Store UUID filter values as compact binaries (#4608)

## Summary
- Store parsed UUID values as 16-byte binaries (rather than 36 byte
strings) inside the sync-service filter/eval representation, converting
back to canonical UUID text at API and SQL output boundaries. This
reduces memory use across the system, but especially in where-clause
filter indexes, such as the equality and subquery index keys, by
avoiding full canonical UUID strings for indexed values.
- Bump the ShapeStatus metadata version so persisted shape comparable
terms created with the old UUID string representation are ignored and
rebuilt under the new representation.
- Add a changeset for `@core/sync-service`.

## Testing
- `mix test test/electric/shape_cache/shape_status_test.exs
test/electric/shape_cache/shape_status/shape_db_test.exs`
- Temporarily patched the standard oracle schema from text/int IDs to
UUID PK/FK columns and ran `mix test --only oracle
test/integration/oracle_property_test.exs` successfully. This patch was
not committed.
- Temporarily ran the larger UUID oracle stress check successfully:
`CHECK_TIMEOUT=60000 SHAPE_COUNT=800 MUTATIONS_PER_TXN=10
TXNS_PER_BATCH=10 BATCH_COUNT=200 SKIP_REPATCH_PREWARM=true mix test
--seed 8 --only oracle test/integration/oracle_property_test.exs`. This
patch was not committed.
- Temporarily added route/subset coverage for `id = $1::uuid` and `id =
CAST($1 AS uuid)` against the `items.id` UUID column; the four targeted
tests passed and were not committed.

Author: robacourt

.changeset/compact-uuid-filter-indexes.md

@@ -0,0 +1,5 @@
+---
+'@core/sync-service': patch
+---
+
+Store parsed UUID values as 16-byte binaries instead of canonical text inside the sync-service filter evaluator. This reduces memory use in where-clause filter indexes, especially equality and subquery index keys, while converting UUIDs back to canonical strings at API and SQL output boundaries. The shape metadata version is bumped so persisted shape comparable terms created with the old UUID string representation are ignored and rebuilt.

packages/sync-service/lib/electric/replication/eval.ex

@@ -72,10 +72,14 @@ defmodule Electric.Replication.Eval do
   for binary protocol encoding.
 
   Most types (integers, floats, booleans, dates, times, etc.) use native Elixir
-  types that Postgrex handles directly. UUID is a notable exception: the eval
-  system stores UUIDs as human-readable strings, but Postgrex expects 16-byte
-  raw binaries.
+  types that Postgrex handles directly. UUIDs are the notable type here because
+  Postgres exposes them as canonical text in decoded rows/API JSON, but Postgrex
+  expects 16-byte raw binaries when sending UUID query parameters. The eval
+  representation stores UUIDs in that compact Postgrex-ready form and converts
+  back to canonical text at type-aware output boundaries.
   """
+  def value_to_postgrex(<<_::128>> = value, :uuid), do: value
+
   def value_to_postgrex(value, :uuid) when is_binary(value) do
     {:ok, bin} = Ecto.UUID.dump(value)
     bin

packages/sync-service/lib/electric/replication/eval/env/known_functions.ex

@@ -94,7 +94,7 @@ defmodule Electric.Replication.Eval.Env.KnownFunctions do
     def bool_out(false), do: "f"
   end
 
-  defpostgres("uuidout(uuid) -> text", delegate: &BasicTypes.noop/1)
+  defpostgres("uuidout(uuid) -> text", delegate: &Casting.uuid_to_string/1)
 
   ## Comparison functions
 

packages/sync-service/lib/electric/replication/eval/sql_generator.ex

@@ -17,6 +17,7 @@ defmodule Electric.Replication.Eval.SqlGenerator do
   """
 
   alias Electric.Replication.Eval.Parser.{Const, Ref, Func, Array, RowExpr}
+  alias Electric.Replication.PostgresInterop.Casting
 
   # PostgreSQL operator precedence (higher number = tighter binding)
   # See: https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-PRECEDENCE
@@ -254,6 +255,15 @@ defmodule Electric.Replication.Eval.SqlGenerator do
   defp to_sql_prec(%Const{value: true}), do: {"true", @prec_atom}
   defp to_sql_prec(%Const{value: false}), do: {"false", @prec_atom}
 
+  defp to_sql_prec(%Const{type: :uuid, value: value}) when is_binary(value) do
+    {uuid_literal(value), @prec_atom}
+  end
+
+  defp to_sql_prec(%Const{type: {:array, type}, value: value}) when is_list(value) do
+    elements = Enum.map_join(value, ", ", &const_list_element_to_sql(&1, type))
+    {"ARRAY[#{elements}]", @prec_atom}
+  end
+
   defp to_sql_prec(%Const{value: value}) when is_binary(value) do
     escaped = String.replace(value, "'", "''")
     {"'#{escaped}'", @prec_atom}
@@ -352,6 +362,18 @@ defmodule Electric.Replication.Eval.SqlGenerator do
     "ARRAY[#{elements}]"
   end
 
+  defp const_list_element_to_sql(value, :uuid) when is_binary(value), do: uuid_literal(value)
+  defp const_list_element_to_sql(nil, _type), do: "NULL"
+
+  defp const_list_element_to_sql(values, {:array, type}) when is_list(values) do
+    elements = Enum.map_join(values, ", ", &const_list_element_to_sql(&1, type))
+    "ARRAY[#{elements}]"
+  end
+
+  defp const_list_element_to_sql(value, _type), do: const_list_element_to_sql(value)
+
+  defp uuid_literal(value), do: "'#{Casting.uuid_to_string(value)}'::uuid"
+
   # Helper for ANY/ALL: extract the operator, left operand, and array right operand
   # from a Func with map_over_array_in_pos set
   defp extract_mapped_operator(%Func{name: name, args: [left, right]}) do

packages/sync-service/lib/electric/replication/postgres_interop/casting.ex

@@ -44,7 +44,12 @@ defmodule Electric.Replication.PostgresInterop.Casting do
 
   def parse_uuid(maybe_uuid) do
     {:ok, value} = Ecto.UUID.dump(maybe_uuid)
-    Ecto.UUID.load!(value)
+    value
+  end
+
+  def uuid_to_string(maybe_uuid) do
+    {:ok, uuid} = Ecto.UUID.cast(maybe_uuid)
+    uuid
   end
 
   def parse_date("epoch"), do: Date.from_iso8601!("1970-01-01")

packages/sync-service/lib/electric/shape_cache/shape_status.ex

@@ -33,7 +33,7 @@ defmodule Electric.ShapeCache.ShapeStatus do
         }
 
   # MUST be updated when Shape.comparable/1 changes.
-  @version 9
+  @version 10
 
   # Tuple format: {handle, hash, snapshot_started, last_read_time, generation}
   @shape_last_used_time_pos 4

packages/sync-service/test/electric/replication/eval/env_test.exs

@@ -3,10 +3,45 @@ defmodule Electric.Replication.Eval.EnvTest do
 
   alias Electric.Replication.Eval.Env
 
-  describe "const_to_pg_string/3" do
+  describe "parse_const/3" do
     setup do
       env = Env.new()
-      %{env: env}
+      uuid = "550e8400-e29b-41d4-a716-446655440000"
+      {:ok, uuid_bytes} = Ecto.UUID.dump(uuid)
+      %{env: env, uuid: uuid, uuid_bytes: uuid_bytes}
+    end
+
+    test "parses uuid values as compact 16-byte binaries", %{
+      env: env,
+      uuid: uuid,
+      uuid_bytes: uuid_bytes
+    } do
+      assert {:ok, ^uuid_bytes} = Env.parse_const(env, uuid, :uuid)
+      assert byte_size(uuid_bytes) == 16
+    end
+
+    test "rejects invalid 16-byte uuid text", %{env: env} do
+      assert :error = Env.parse_const(env, "abcdefghijklmnop", :uuid)
+    end
+
+    test "parses uuid arrays as compact 16-byte binaries", %{
+      env: env,
+      uuid: uuid,
+      uuid_bytes: uuid_bytes
+    } do
+      assert {:ok, [^uuid_bytes]} = Env.parse_const(env, "{#{uuid}}", {:array, :uuid})
+    end
+
+    test "rejects invalid 16-byte uuid text inside arrays", %{env: env} do
+      assert :error = Env.parse_const(env, "{abcdefghijklmnop}", {:array, :uuid})
+    end
+  end
+
+  describe "const_to_pg_string/3" do
+    setup do
+      uuid = "550e8400-e29b-41d4-a716-446655440000"
+      {:ok, uuid_bytes} = Ecto.UUID.dump(uuid)
+      %{env: Env.new(), uuid: uuid, uuid_bytes: uuid_bytes}
     end
 
     test "converts text type values as-is", %{env: env} do
@@ -62,9 +97,12 @@ defmodule Electric.Replication.Eval.EnvTest do
       assert Env.const_to_pg_string(env, timestamp, :timestamp) == "2025-01-15T14:30:00"
     end
 
-    test "converts uuid type as-is (noop)", %{env: env} do
-      uuid = "550e8400-e29b-41d4-a716-446655440000"
-      assert Env.const_to_pg_string(env, uuid, :uuid) == uuid
+    test "converts compact uuid values back to canonical text", %{
+      env: env,
+      uuid: uuid,
+      uuid_bytes: uuid_bytes
+    } do
+      assert Env.const_to_pg_string(env, uuid_bytes, :uuid) == uuid
     end
 
     test "converts simple arrays of integers", %{env: env} do

packages/sync-service/test/electric/replication/eval/runner_test.exs

@@ -28,6 +28,14 @@ defmodule Electric.Replication.Eval.RunnerTest do
                  "null_int" => nil
                })
     end
+
+    test "parses uuid refs as compact 16-byte binaries" do
+      uuid = "550e8400-e29b-41d4-a716-446655440000"
+      {:ok, uuid_bytes} = Ecto.UUID.dump(uuid)
+
+      assert {:ok, %{["id"] => ^uuid_bytes}} =
+               Runner.record_to_ref_values(%{["id"] => :uuid}, %{"id" => uuid})
+    end
   end
 
   describe "execute/2" do

packages/sync-service/test/electric/replication/eval/sql_generator_test.exs

@@ -412,6 +412,14 @@ defmodule Electric.Replication.Eval.SqlGeneratorTest do
       assert SqlGenerator.to_sql(%Const{value: "it's"}) == "'it''s'"
     end
 
+    test "uuid" do
+      uuid = "550e8400-e29b-41d4-a716-446655440000"
+      {:ok, uuid_bytes} = Ecto.UUID.dump(uuid)
+
+      assert SqlGenerator.to_sql(%Const{type: :uuid, value: uuid_bytes}) ==
+               "'#{uuid}'::uuid"
+    end
+
     test "integer" do
       assert SqlGenerator.to_sql(%Const{value: 42}) == "42"
     end
@@ -436,6 +444,15 @@ defmodule Electric.Replication.Eval.SqlGeneratorTest do
       assert SqlGenerator.to_sql(ast) == "ARRAY['a', 'b']"
     end
 
+    test "constant-folded uuid array" do
+      uuid = "550e8400-e29b-41d4-a716-446655440000"
+      {:ok, uuid_bytes} = Ecto.UUID.dump(uuid)
+
+      ast = %Const{type: {:array, :uuid}, value: [uuid_bytes]}
+
+      assert SqlGenerator.to_sql(ast) == "ARRAY['#{uuid}'::uuid]"
+    end
+
     test "empty array" do
       ast = %Array{elements: []}
       assert SqlGenerator.to_sql(ast) == "ARRAY[]"

packages/sync-service/test/electric/shapes/filter_test.exs

@@ -90,6 +90,38 @@ defmodule Electric.Shapes.FilterTest do
       assert Filter.affected_shapes(filter, insert) == MapSet.new(["s2"])
     end
 
+    test "returns shapes affected by uuid equality using compact index keys" do
+      uuid1 = "550e8400-e29b-41d4-a716-446655440000"
+      uuid2 = "550e8400-e29b-41d4-a716-446655440001"
+      {:ok, uuid1_bytes} = Ecto.UUID.dump(uuid1)
+
+      inspector =
+        StubInspector.new(
+          tables: ["uuid_table"],
+          columns: [%{name: "id", type: "uuid", pk_position: 0}]
+        )
+
+      filter =
+        Filter.new()
+        |> Filter.add_shape(
+          "s1",
+          Shape.new!("uuid_table", where: "id = '#{uuid1}'", inspector: inspector)
+        )
+        |> Filter.add_shape(
+          "s2",
+          Shape.new!("uuid_table", where: "id = '#{uuid2}'", inspector: inspector)
+        )
+
+      index_entries = :ets.tab2list(filter.eq_index_table)
+
+      assert Enum.any?(index_entries, &match?({{_, "id", ^uuid1_bytes}, _}, &1))
+      refute Enum.any?(index_entries, &match?({{_, "id", ^uuid1}, _}, &1))
+
+      insert = %NewRecord{relation: {"public", "uuid_table"}, record: %{"id" => uuid1}}
+
+      assert Filter.affected_shapes(filter, insert) == MapSet.new(["s1"])
+    end
+
     test "returns shapes affected by delete" do
       filter =
         Filter.new()