fix(tests): self-install eql_v3 in property suites so CI shards pass (CIP-3141)

The fixture/e2e property oracles connect via connect_pool() to the base
DATABASE_URL database, not through #[sqlx::test]'s migrated per-test scratch
DBs. Under the sharded build-once-archive CI, a shard's base DB is a stock
Postgres with no EQL installed (only build-archive ran `sqlx migrate run`,
against a different Postgres), so every `::eql_v3.<T>_eq` cast raised
`schema "eql_v3" does not exist`. The old per-version CI happened to install
EQL into the base DB via test:sqlx:prep's migrate step, which the suites
silently relied on; the rebase onto the new CI removed that, turning the
suites red. (The failures looked type-specific — int2/int8/text/date/
timestamptz — but that was just nextest fail-fast + shard distribution; the
error is type-agnostic, int4 included.)

Make the suites self-sufficient: ensure_eql_installed() applies the embedded
001_install_eql.sql installer to the connected DB on first use, guarded by a
once-per-process async mutex and an `eql_v3.int4_eq` presence check (so a
developer's pre-installed local DB is left untouched — the installer is not
idempotent). The installer is include_str!'d in the test target (mod.rs), the
same archive-travel mechanism the fixture corpus uses, kept out of the lib so
clippy/test:crates contexts don't need the generated file.

Also stop swallowing the real Postgres error: the proptest runners rendered
anyhow errors with {e}/to_string(), dropping the cause chain and leaving only
the "pair query: SELECT …" context line — which is what made this opaque in
CI. Render with {e:#} to keep the full chain.

Verified: against a fresh DB with no eql_v3, all 11 fixture_oracle tests
(including the ones that failed in CI) now pass and the surface is installed
on demand; against an already-installed DB the presence check skips the install.

Author: tobyhede

tests/sqlx/src/property.rs

@@ -22,6 +22,52 @@ use tokio::sync::Mutex;
 /// `fixtures.eql_v2_<T>` table exactly once.
 static FIXTURE_LOADED: OnceLock<Mutex<HashSet<&'static str>>> = OnceLock::new();
 
+/// Per-process guard ensuring the EQL surface (the `eql_v3` schema the oracle
+/// queries cast to) is installed into the connected DB exactly once.
+static EQL_INSTALLED: OnceLock<Mutex<bool>> = OnceLock::new();
+
+/// Ensure the EQL surface (the `eql_v3` schema + scalar domains/operators the
+/// oracle queries cast to) is present in the DB behind `pool`.
+///
+/// The property suites connect via `connect_pool()` to the base test database
+/// (`DATABASE_URL`), NOT through `#[sqlx::test]`'s migrated per-test scratch
+/// DBs. In a CI shard that base DB is a stock Postgres with no EQL installed —
+/// only the `build-archive` job ran `sqlx migrate run`, and against a different
+/// Postgres — so every `::eql_v3.<T>_eq` cast would raise
+/// `schema "eql_v3" does not exist`. This installs the surface on demand so the
+/// suites are self-sufficient regardless of where they run (CI shard, local,
+/// fork), instead of silently depending on a pre-installed base DB.
+///
+/// `install_sql` is the EQL installer (`migrations/001_install_eql.sql`),
+/// `include_str!`-embedded into the test binary at compile time (see
+/// `property/mod.rs`) so it travels inside the prebuilt nextest archive — the
+/// same mechanism the fixture corpus uses. A process-wide async mutex
+/// guarantees exactly-once execution across the parallel proptest threads, and
+/// a presence check (`eql_v3.int4_eq`) skips the install when the DB already
+/// has the surface (a developer's pre-installed local DB), where re-running the
+/// non-idempotent installer would error on duplicate objects.
+pub async fn ensure_eql_installed(pool: &PgPool, install_sql: &str) -> Result<()> {
+    let guard = EQL_INSTALLED.get_or_init(|| Mutex::new(false));
+    let mut installed = guard.lock().await;
+    if *installed {
+        return Ok(());
+    }
+    // Presence check: skip the installer if the surface is already there. int4
+    // is the reference scalar type and is always part of the surface.
+    let present: bool = sqlx::query_scalar("SELECT to_regtype('eql_v3.int4_eq') IS NOT NULL")
+        .fetch_one(pool)
+        .await
+        .context("probing for an existing eql_v3 install")?;
+    if !present {
+        sqlx::raw_sql(install_sql)
+            .execute(pool)
+            .await
+            .context("installing the EQL surface into the property-test DB")?;
+    }
+    *installed = true;
+    Ok(())
+}
+
 /// Materialise the committed fixture corpus (real ciphertext) for `T` into the connected DB.
 ///
 /// The fixture `.sql` files (`tests/sqlx/fixtures/eql_v2_<T>.sql`) are normally

tests/sqlx/tests/encrypted_domain/property/e2e_oracle.rs

@@ -11,7 +11,9 @@ use anyhow::Result;
 use eql_tests::fixtures::cipherstash::{column_config_for, encrypt_store};
 use eql_tests::fixtures::eql_plaintext::{Cast, EqlPlaintext};
 use eql_tests::fixtures::index_kind::IndexKind;
-use eql_tests::property::{assert_eq_oracle, assert_ord_oracle, connect_pool, Row};
+use eql_tests::property::{
+    assert_eq_oracle, assert_ord_oracle, connect_pool, ensure_eql_installed, Row,
+};
 use eql_tests::scalar_domains::ScalarType;
 use eql_tests::scalar_domains::Variant;
 use proptest::prelude::*;
@@ -64,6 +66,9 @@ where
         .enable_all()
         .build()?;
     let pool: PgPool = rt.block_on(connect_pool())?;
+    // The base DB this pool connects to is not migrated by `#[sqlx::test]`; in a
+    // CI shard it has no `eql_v3` surface, so install it before any cast/query.
+    rt.block_on(ensure_eql_installed(&pool, super::EQL_INSTALL_SQL))?;
 
     // Shrinking is disabled for the e2e suite: every failed shrink attempt would
     // trigger another ZeroKMS batch, and ciphertext cannot be meaningfully
@@ -91,7 +96,9 @@ where
             values.push(dup1);
             let rows = rt
                 .block_on(encrypt_rows::<T>(table, cast, &values))
-                .map_err(|e| TestCaseError::fail(format!("encrypt: {e}")))?;
+                // `{e:#}` keeps anyhow's full cause chain (the underlying error),
+                // which a plain `{e}` would drop.
+                .map_err(|e| TestCaseError::fail(format!("encrypt: {e:#}")))?;
             rt.block_on(async {
                 assert_eq_oracle::<T>(&pool, &rows).await?;
                 if ordered {
@@ -100,7 +107,7 @@ where
                 }
                 Ok::<_, anyhow::Error>(())
             })
-            .map_err(|e| TestCaseError::fail(format!("oracle: {e}")))?;
+            .map_err(|e| TestCaseError::fail(format!("oracle: {e:#}")))?;
             Ok(())
         })
         .map_err(|e| anyhow::anyhow!("e2e property failed: {e}"))

tests/sqlx/tests/encrypted_domain/property/fixture_oracle.rs

@@ -10,7 +10,8 @@
 
 use anyhow::Result;
 use eql_tests::property::{
-    assert_eq_oracle, assert_ord_oracle, connect_pool, ensure_fixture_loaded, Row,
+    assert_eq_oracle, assert_ord_oracle, connect_pool, ensure_eql_installed, ensure_fixture_loaded,
+    Row,
 };
 use eql_tests::scalar_domains::{ScalarType, Variant};
 use proptest::prelude::*;
@@ -63,6 +64,9 @@ fn embedded_fixture_sql<T: ScalarType>() -> &'static str {
 /// Ensures the corpus is present in the shared DB first (it lives in
 /// `#[sqlx::test]`'s ephemeral DBs by default, not the pool we connect to).
 async fn load_fixture_rows<T: ScalarType>(pool: &PgPool) -> Result<Vec<Row<T>>> {
+    // The base DB this pool connects to is not migrated by `#[sqlx::test]`; in a
+    // CI shard it has no `eql_v3` surface, so install it before any cast/query.
+    ensure_eql_installed(pool, super::EQL_INSTALL_SQL).await?;
     ensure_fixture_loaded::<T>(pool, embedded_fixture_sql::<T>()).await?;
     let sql = format!(
         "SELECT plaintext, payload::text FROM {} ORDER BY id",
@@ -118,7 +122,10 @@ where
         .run(&strategy, |idxs| {
             let corpus = pick(&all, &idxs);
             rt.block_on(oracle(pool.clone(), corpus))
-                .map_err(|e| TestCaseError::fail(e.to_string()))?;
+                // `{e:#}` renders anyhow's full cause chain inline; plain
+                // `to_string()` drops it, hiding the real Postgres error (e.g.
+                // `schema "eql_v3" does not exist`) behind only the context line.
+                .map_err(|e| TestCaseError::fail(format!("{e:#}")))?;
             Ok(())
         })
         .map_err(|e| anyhow::anyhow!("fixture property failed: {e}"))

tests/sqlx/tests/encrypted_domain/property/mod.rs

@@ -4,6 +4,19 @@
 //! `scalars::<X>::` test-name prefix, so a `scalars::property::…` test would be
 //! mis-read as a scalar type and break the catalog cross-check.
 
+/// The EQL installer (`migrations/001_install_eql.sql`, the full release),
+/// embedded at compile time so the property suites can install the `eql_v3`
+/// surface into their base DB on demand (see `property::ensure_eql_installed`).
+/// Same embed-into-the-archive rationale as the per-type fixture corpus in
+/// `fixture_oracle.rs`: the file is produced by `test:sqlx:prep` before the
+/// nextest archive is built, and the CI shards run from that archive without a
+/// checkout of the gitignored generated SQL. The path resolves against the
+/// `eql_tests` crate root (`tests/sqlx`).
+pub(crate) const EQL_INSTALL_SQL: &str = include_str!(concat!(
+    env!("CARGO_MANIFEST_DIR"),
+    "/migrations/001_install_eql.sql"
+));
+
 // NULL / blocker / CHECK-constraint unit tests.
 mod edge_cases;
 // fixture suite: oracle over the committed fixture corpus (real ciphertext).