diff --git a/labs/converged-database-lab/.gitignore b/labs/converged-database-lab/.gitignore index ea6ec7b..62008e8 100644 --- a/labs/converged-database-lab/.gitignore +++ b/labs/converged-database-lab/.gitignore @@ -5,3 +5,4 @@ validator/results.json validator/.venv/ docker/.data/ *.pyc +tasks/ diff --git a/labs/converged-database-lab/README.md b/labs/converged-database-lab/README.md index b6365f2..2148651 100644 --- a/labs/converged-database-lab/README.md +++ b/labs/converged-database-lab/README.md @@ -1,6 +1,6 @@ # converged-database-lab -![validate](https://github.com/oracle-devrel/oracle-umt-developer-hub/actions/workflows/converged-database-lab.yml/badge.svg) +![validate](https://github.com/rhoulihan/converged-database-lab/actions/workflows/validate.yml/badge.svg) Runnable proofs for the converged-database article series — every technical claim in the articles executes here, nightly, against a free container. @@ -80,8 +80,10 @@ All modules run against one deterministically seeded domain. Seeds use an LCG (s | `idx_orders_customer` | B-tree | Customer order range scans | | `idx_items_product` | B-tree | Product-level aggregations | | `idx_tickets_customer` | B-tree | Per-customer ticket lookups | -| `ticket_text_idx` | Oracle Text (SEARCH INDEX) | Full-text keyword search on ticket bodies | -| `ticket_vec_idx` | Vector IVF (NEIGHBOR PARTITIONS, COSINE) | Approximate nearest-neighbor on embeddings | +| `ticket_text_idx` | Oracle Text (SEARCH INDEX, SYNC ON COMMIT) | Transactional keyword search on ticket subjects (module 02's read-after-write proof; body text search belongs to the HVI) | +| `ticket_vec_idx` | Vector IVF (NEIGHBOR PARTITIONS, COSINE) | Approximate nearest-neighbor on 8-dim demo embeddings | +| `tickets_bodyvec_ivf` | Vector IVF (NEIGHBOR PARTITIONS, COSINE) | ANN on the real 384-dim `body_vec` embeddings (module 03) | +| `tickets_hvi` | Hybrid Vector Index (Text + vector, one domain index) | Fused keyword + semantic search on ticket bodies (module 03) | **Note on embeddings:** The 8-dim vectors are deterministic demo values generated by the LCG seed. All engine behaviors demonstrated (distance functions, vector indexes, transactions, cross-model joins) are dimension-independent. Module 03 documents the optional real-model flow using `DBMS_VECTOR.LOAD_ONNX_MODEL` with all-MiniLM-L12-v2. @@ -95,7 +97,7 @@ All modules run against one deterministically seeded domain. Seeds use an LCG (s |---|------|--------|--------|------------|-------| | 01 | `what-is-a-converged-database` | Live | 5 | 20 | [README](modules/01-what-is-a-converged-database/README.md) · [article](https://blogs.oracle.com/developers/what-is-a-converged-database-definition-five-tests-and-ai-use-cases) | | 02 | `converged-vs-multi-model` | Live | 3 | 18 | [README](modules/02-converged-vs-multi-model/README.md) · [article](https://blogs.oracle.com/developers/converged-database-vs-multi-model-database-whats-the-difference) | -| 03 | `vector-vs-converged` | On branch, CI-green — article in draft | 4 | 19 | [README](https://github.com/oracle-devrel/oracle-umt-developer-hub/blob/article/03-vector-vs-converged/labs/converged-database-lab/modules/03-vector-vs-converged/README.md) · [branch](https://github.com/oracle-devrel/oracle-umt-developer-hub/tree/article/03-vector-vs-converged) · article at publish | +| 03 | `vector-vs-converged` | On main, validated — article in draft | 4 | 23 | [README](modules/03-vector-vs-converged/README.md) · [branch](https://github.com/oracle-devrel/oracle-umt-developer-hub/tree/article/03-vector-vs-converged) · article at publish | | 04 | `ai-agents-enterprise-data` | On branch, CI-green — article in draft | 3 | 16 | [README](https://github.com/oracle-devrel/oracle-umt-developer-hub/blob/article/04-ai-agents-enterprise-data/labs/converged-database-lab/modules/04-ai-agents-enterprise-data/README.md) · [branch](https://github.com/oracle-devrel/oracle-umt-developer-hub/tree/article/04-ai-agents-enterprise-data) · article at publish | | 05 | `json-graph-vector-relational` | On branch, CI-green — article in draft | 4 | 26 | [README](https://github.com/oracle-devrel/oracle-umt-developer-hub/blob/article/05-json-graph-vector-relational/labs/converged-database-lab/modules/05-json-graph-vector-relational/README.md) · [branch](https://github.com/oracle-devrel/oracle-umt-developer-hub/tree/article/05-json-graph-vector-relational) · article at publish | | 06 | `secure-agent-access` | On branch, CI-green — article in draft | 4 | 7 | [README](https://github.com/oracle-devrel/oracle-umt-developer-hub/blob/article/06-secure-agent-access/labs/converged-database-lab/modules/06-secure-agent-access/README.md) · [branch](https://github.com/oracle-devrel/oracle-umt-developer-hub/tree/article/06-secure-agent-access) · article at publish | diff --git a/labs/converged-database-lab/docker-compose.yml b/labs/converged-database-lab/docker-compose.yml index c8b8610..69d0e7f 100644 --- a/labs/converged-database-lab/docker-compose.yml +++ b/labs/converged-database-lab/docker-compose.yml @@ -17,7 +17,10 @@ services: test: ['CMD-SHELL', '/opt/oracle/healthcheck.sh && curl -so /dev/null -w "%{http_code}" http://localhost:8181/ | grep -q "302\|200"'] interval: 15s timeout: 10s - retries: 40 + # 80 retries: first boot now embeds 10,000 ticket bodies in-database and + # builds the HVI over them (~7-9 min of init on top of DB create), so the + # container must not be declared unhealthy before init can finish. + retries: 80 start_period: 120s deploy: resources: diff --git a/labs/converged-database-lab/docker/Dockerfile.oracle b/labs/converged-database-lab/docker/Dockerfile.oracle index b55193a..324e0dc 100644 --- a/labs/converged-database-lab/docker/Dockerfile.oracle +++ b/labs/converged-database-lab/docker/Dockerfile.oracle @@ -9,10 +9,34 @@ RUN chown -R oracle:oinstall /etc/ords # mongosh: used to verify and exercise the MongoDB API (port 27017) from inside # the container — the validator harness runs *.js scripts through it. -RUN printf '[mongodb-org-8.0]\nname=MongoDB Repository\nbaseurl=https://repo.mongodb.org/yum/redhat/8/mongodb-org/8.0/x86_64/\ngpgcheck=1\nenabled=1\ngpgkey=https://pgp.mongodb.com/server-8.0.asc\n' > /etc/yum.repos.d/mongodb-org-8.0.repo \ +# Arch-aware repo URL: CI builds on x86_64; local builds on Apple-silicon (aarch64) +# podman machines get MongoDB's aarch64 el8 packages instead of failing dependency +# resolution against the x86_64 repo. +RUN printf '[mongodb-org-8.0]\nname=MongoDB Repository\nbaseurl=https://repo.mongodb.org/yum/redhat/8/mongodb-org/8.0/%s/\ngpgcheck=1\nenabled=1\ngpgkey=https://pgp.mongodb.com/server-8.0.asc\n' "$(uname -m)" > /etc/yum.repos.d/mongodb-org-8.0.repo \ && microdnf install -y mongodb-mongosh \ && microdnf clean all +# Bake Oracle's prebuilt augmented all-MiniLM-L12-v2 ONNX model into the image so +# every fresh build (and CI) loads identical in-DB embeddings deterministically — +# no network dependency at init time. This is the official model published by +# Oracle Machine Learning: the 26ai docs "ONNX Pipeline Models: Text Embedding" +# page redirects (/pls/topic/lookup?...&id=oml_ai_models_object_storage) to the +# OML-ai-models object-storage bucket whose index lists this exact URL. The zip +# (~122.6 MB) unpacks to all_MiniLM_L12_v2.onnx (~133.3 MB, 384-dim output), +# loaded by docker/init/08-vector-model.sql via DBMS_VECTOR.LOAD_ONNX_MODEL. +# Placed AFTER the ORDS + mongosh layers so their (slow) cache layers are reused +# when only the model layer changes. Fails the build loudly if the URL stops +# resolving rather than baking a broken image. +ARG MINILM_URL=https://adwc4pm.objectstorage.us-ashburn-1.oci.customer-oci.com/p/TtH6hL2y25EypZ0-rrczRZ1aXp7v1ONbRBfCiT-BDBN8WLKQ3lgyW6RxCfIFLdA6/n/adwc4pm/b/OML-ai-models/o/all_MiniLM_L12_v2_augmented.zip +RUN microdnf install -y unzip \ + && microdnf clean all \ + && mkdir -p /opt/oracle/models \ + && curl -fsSL -o /tmp/minilm.zip "$MINILM_URL" \ + && unzip -o /tmp/minilm.zip -d /opt/oracle/models \ + && test -s /opt/oracle/models/all_MiniLM_L12_v2.onnx \ + && rm -f /tmp/minilm.zip \ + && chown -R oracle:oinstall /opt/oracle/models + COPY init/ /container-entrypoint-initdb.d/ COPY scripts/entrypoint.sh /opt/oracle/scripts/entrypoint.sh diff --git a/labs/converged-database-lab/docker/init/05-text-vector.sql b/labs/converged-database-lab/docker/init/05-text-vector.sql index ddd6d49..b5e0f3c 100644 --- a/labs/converged-database-lab/docker/init/05-text-vector.sql +++ b/labs/converged-database-lab/docker/init/05-text-vector.sql @@ -1,14 +1,23 @@ ALTER SESSION SET CONTAINER = FREEPDB1; ALTER SESSION SET CURRENT_SCHEMA = lab_user; --- Oracle Text index on ticket bodies: keyword search in the same engine/transaction. --- SYNC (ON COMMIT): the index syncs inside the committing transaction, so a --- committed row is immediately findable via CONTAINS — transactional --- read-after-write search (module 02 proves this). The 26ai Free default for --- CREATE SEARCH INDEX is MAINTENANCE AUTO with deferred background sync --- (ctx_user_indexes showed IDX_SYNC_TYPE = MANUAL), under which a probe row was --- NOT visible to CONTAINS immediately after COMMIT. -CREATE SEARCH INDEX ticket_text_idx ON support_tickets (body) +-- Oracle Text index on ticket SUBJECTS: keyword search in the same +-- engine/transaction. SYNC (ON COMMIT): the index syncs inside the committing +-- transaction, so a committed row is immediately findable via CONTAINS — +-- transactional read-after-write search (module 02 proves this). The 26ai Free +-- default for CREATE SEARCH INDEX is MAINTENANCE AUTO with deferred background +-- sync (ctx_user_indexes showed IDX_SYNC_TYPE = MANUAL), under which a probe +-- row was NOT visible to CONTAINS immediately after COMMIT. +-- +-- MERGE RECONCILIATION (modules 02 + 03 together on main): this index lives on +-- SUBJECT here, not BODY. Module 03's Hybrid Vector Index owns body text search +-- (an HVI's text component is the same indextype as a SEARCH index — two cannot +-- share a column, ORA-29879), and the HVI's text component does NOT sync on +-- commit (verified empirically 2026-07-24: a committed probe row was not +-- CONTAINS-findable through the HVI). Module 02's read-after-write proof +-- therefore probes the subject column on main; the article/02 BRANCH keeps the +-- body form that the published article's snippets byte-match. +CREATE SEARCH INDEX ticket_text_idx ON support_tickets (subject) PARAMETERS ('SYNC (ON COMMIT)'); -- Vector index. IVF (NEIGHBOR PARTITIONS) — works within Free-tier memory without diff --git a/labs/converged-database-lab/docker/init/06-seed.sql b/labs/converged-database-lab/docker/init/06-seed.sql index c1d0c37..dc9007e 100644 --- a/labs/converged-database-lab/docker/init/06-seed.sql +++ b/labs/converged-database-lab/docker/init/06-seed.sql @@ -4,6 +4,13 @@ ALTER SESSION SET CURRENT_SCHEMA = lab_user; DECLARE -- All item declarations must precede subprogram bodies (PLS-00103 otherwise). seed NUMBER := 42; + -- Ticket volume. 10,000 tickets is deliberately past the point where the + -- cost-based optimizer chooses the IVF vector index over exact full-scan + -- distance for module 03's filtered FETCH APPROX query (the crossover + -- arrives by roughly 3,000 rows on the Free container with fresh stats) — + -- so the captured plan shows the index + in-plan predicate evaluation, not + -- a toy-size full scan. + c_tickets CONSTANT PLS_INTEGER := 10000; v_vec VARCHAR2(400); v_norm NUMBER; TYPE t_f IS TABLE OF NUMBER INDEX BY PLS_INTEGER; @@ -18,6 +25,56 @@ DECLARE v_prod PLS_INTEGER; v_qty PLS_INTEGER; v_ts TIMESTAMP; + v_cat PLS_INTEGER; + v_body VARCHAR2(2000); + -- Enriched ticket-body corpus for tickets 301..c_tickets: 6 intros x 8 issue + -- categories x 4 details x 50 products x 5 closers ~= 48k distinct bodies, so + -- the 384-dim embeddings form real semantic clusters instead of 4 duplicated + -- points (matters for a meaningful IVF index and hybrid-search demo). + TYPE t_v IS TABLE OF VARCHAR2(400); + intro t_v := t_v('Customer reports an issue with their recent experience.', + 'Contacting support after several attempts to resolve this alone.', + 'Opening a ticket on behalf of the account holder.', + 'Follow-up on an earlier conversation with the support team.', + 'New request submitted through the help center.', + 'Escalated from chat: the problem is still not resolved.'); + refund t_v := t_v('The package arrived damaged and a refund is requested.', + 'The box was crushed in transit and the item inside is damaged beyond use.', + 'Item arrived with a cracked casing; customer asks for their money back.', + 'Shipment was left in the rain and the contents are damaged and unusable.'); + login t_v := t_v('Login fails with an authentication error after reset.', + 'Password reset completed but sign-in still rejects the new credentials.', + 'Two-factor prompt never arrives and access to the account is blocked.', + 'Account is locked out after the security update and recovery does not work.'); + shipdl t_v := t_v('The order has not arrived within the promised window.', + 'Tracking has shown no movement for a week and the delivery date passed.', + 'Carrier marked the parcel delivered but nothing has arrived.', + 'Estimated delivery keeps slipping and no update has been provided.'); + warr t_v := t_v('Customer asks whether the warranty covers accidental damage.', + 'Requesting warranty service for a unit that stopped powering on.', + 'Screen developed dead pixels within the coverage period.', + 'Asking how to start a warranty claim for a failed component.'); + billing t_v := t_v('The statement shows a duplicate charge for a single order.', + 'A promotional discount was not applied at checkout.', + 'The invoice total does not match the order confirmation email.', + 'Customer was billed after cancelling the subscription.'); + exch t_v := t_v('Requesting an exchange for a different size of the same product.', + 'The wrong color variant was delivered and a swap is requested.', + 'Received a different model than the one ordered.', + 'Asking to exchange a gift for store credit instead.'); + install t_v := t_v('The setup guide does not match the ports on the device.', + 'Firmware update fails at forty percent every attempt.', + 'The companion app cannot discover the device during pairing.', + 'Installation completes but the device reboots continuously.'); + perf t_v := t_v('The device becomes very slow after a few hours of use.', + 'Battery drains overnight even when the unit is idle.', + 'Audio cuts out intermittently during playback.', + 'The connection drops whenever a second device joins.'); + closing t_v := t_v('Customer asks for a response today.', + 'This is the second time the issue has been reported.', + 'The customer mentions they are a long-time subscriber.', + 'A callback number has been left on the account.', + 'No workaround has been found so far.'); FUNCTION nxt RETURN NUMBER IS -- LCG: deterministic across runs BEGIN seed := MOD(seed * 1103515245 + 12345, 2147483648); @@ -25,6 +82,14 @@ DECLARE END; FUNCTION pick(n IN PLS_INTEGER) RETURN PLS_INTEGER IS BEGIN RETURN TRUNC(nxt * n) + 1; END; + FUNCTION cat_detail(p_cat PLS_INTEGER) RETURN VARCHAR2 IS + BEGIN + RETURN CASE p_cat + WHEN 1 THEN refund (pick(4)) WHEN 2 THEN login(pick(4)) + WHEN 3 THEN shipdl (pick(4)) WHEN 4 THEN warr (pick(4)) + WHEN 5 THEN billing(pick(4)) WHEN 6 THEN exch (pick(4)) + WHEN 7 THEN install(pick(4)) ELSE perf(pick(4)) END; + END; BEGIN FOR i IN 1..200 LOOP INSERT INTO customers (email, full_name, segment) @@ -95,6 +160,9 @@ BEGIN UPDATE orders SET total_amount = ROUND(v_total,2) WHERE order_id = i; END LOOP; + -- Tickets 1..300: the ORIGINAL 4-template corpus, byte-for-byte (module 01's + -- ticket #1 proofs and module 03's read-after-write probe depend on ticket #1 + -- keeping its "Login fails with an authentication error after reset." body). FOR i IN 1..300 LOOP v_norm := 0; FOR d IN 1..8 LOOP f(d) := nxt*2 - 1; v_norm := v_norm + f(d)*f(d); END LOOP; @@ -119,6 +187,38 @@ BEGIN CASE WHEN MOD(i,3)=0 THEN 'closed' WHEN MOD(i,7)=0 THEN 'pending' ELSE 'open' END, TO_VECTOR(v_vec, 8, FLOAT32)); END LOOP; + + -- Tickets 301..c_tickets: the enriched scaled corpus. The LCG is re-seeded to + -- a fixed value here so this block is deterministic regardless of how the + -- upstream seeding evolves — same bodies on every fresh build. + seed := 342; + FOR i IN 301..c_tickets LOOP + v_cat := pick(8); + v_cust := pick(200); + v_body := intro(pick(6))||' '||cat_detail(v_cat)|| + ' Relates to Product '||pick(50)||'. '||closing(pick(5)); + v_norm := 0; + FOR d IN 1..8 LOOP f(d) := nxt*2 - 1; v_norm := v_norm + f(d)*f(d); END LOOP; + v_norm := SQRT(v_norm); + v_vec := '['; + FOR d IN 1..8 LOOP + v_vec := v_vec || TO_CHAR(ROUND(f(d)/v_norm, 6),'FM990.999999') || CASE WHEN d<8 THEN ',' END; + END LOOP; + v_vec := v_vec || ']'; + INSERT INTO support_tickets (customer_id, subject, body, status, embedding) + VALUES (v_cust, + CASE v_cat WHEN 1 THEN 'Refund request for damaged item' + WHEN 2 THEN 'Cannot sign in after password reset' + WHEN 3 THEN 'Shipping delay on recent order' + WHEN 4 THEN 'Warranty question for product' + WHEN 5 THEN 'Billing discrepancy on statement' + WHEN 6 THEN 'Exchange request for recent order' + WHEN 7 THEN 'Device setup and installation issue' + ELSE 'Performance problem with product' END || ' #'||i, + v_body, + CASE WHEN MOD(i,3)=0 THEN 'closed' WHEN MOD(i,7)=0 THEN 'pending' ELSE 'open' END, + TO_VECTOR(v_vec, 8, FLOAT32)); + END LOOP; COMMIT; END; / diff --git a/labs/converged-database-lab/docker/init/08-vector-model.sql b/labs/converged-database-lab/docker/init/08-vector-model.sql new file mode 100644 index 0000000..1fbb1f0 --- /dev/null +++ b/labs/converged-database-lab/docker/init/08-vector-model.sql @@ -0,0 +1,97 @@ +ALTER SESSION SET CONTAINER = FREEPDB1; + +-- In-database embeddings infrastructure (foundational; used by the AI articles). +-- Runs once on first boot (gvenzl initdb), after 01-07. Loads Oracle's prebuilt +-- augmented all-MiniLM-L12-v2 ONNX model (baked into the image at /opt/oracle/models +-- by docker/Dockerfile.oracle), embeds all 10,000 ticket bodies into a real +-- VECTOR(384) column, builds an IVF vector index on it, and gathers optimizer +-- stats so the CBO's index-vs-scan choice is deterministic. Idempotent: every +-- CREATE is guarded so a recreated container over a persistent volume re-runs +-- cleanly. +-- +-- Grants used by the AI modules beyond the base 01-lab-user.sql set: +-- CREATE MINING MODEL — DBMS_VECTOR.LOAD_ONNX_MODEL registers an ONNX model +-- READ ON ONNX_MODELS — read the baked .onnx from the image directory +-- EXECUTE DBMS_VECTOR — VECTOR_EMBEDDING / model load +-- EXECUTE DBMS_RLS — VPD policies (permission-aware retrieval proofs) +-- CREATE/DROP ANY CONTEXT — application context for VPD tenant/user predicates +GRANT CREATE MINING MODEL TO lab_user; +GRANT CREATE ANY CONTEXT TO lab_user; +GRANT DROP ANY CONTEXT TO lab_user; +GRANT EXECUTE ON DBMS_VECTOR TO lab_user; +GRANT EXECUTE ON DBMS_RLS TO lab_user; + +-- Directory over the baked model path; READ for lab_user so the load can open it. +CREATE OR REPLACE DIRECTORY ONNX_MODELS AS '/opt/oracle/models'; +GRANT READ ON DIRECTORY ONNX_MODELS TO lab_user; + +ALTER SESSION SET CURRENT_SCHEMA = lab_user; + +-- Load the model into the lab_user schema as MINILM_L12 so VECTOR_EMBEDDING +-- (MINILM_L12 USING AS data) resolves for lab_user. The augmented model +-- carries its own tokenizer + pooling, so the metadata only needs to declare the +-- embedding function and the input attribute name (DATA, matching the AS data +-- alias used everywhere downstream). DROP first for idempotent re-runs. +DECLARE + v_model_path VARCHAR2(400); +BEGIN + BEGIN + DBMS_VECTOR.DROP_ONNX_MODEL(model_name => 'MINILM_L12', force => TRUE); + EXCEPTION WHEN OTHERS THEN NULL; /* not yet loaded — nothing to drop */ + END; + DBMS_VECTOR.LOAD_ONNX_MODEL( + directory => 'ONNX_MODELS', + file_name => 'all_MiniLM_L12_v2.onnx', + model_name => 'MINILM_L12', + metadata => JSON('{"function":"embedding","embeddingOutput":"embedding","input":{"input":["DATA"]}}')); +END; +/ + +-- Real 384-dim embedding column. The module-01 VECTOR(8) `embedding` column and +-- its IVF index (ticket_vec_idx) are LEFT UNTOUCHED. Guarded so a recreated +-- container does not error on the already-present column. +DECLARE + v_exists NUMBER; +BEGIN + SELECT COUNT(*) INTO v_exists FROM user_tab_columns + WHERE table_name = 'SUPPORT_TICKETS' AND column_name = 'BODY_VEC'; + IF v_exists = 0 THEN + EXECUTE IMMEDIATE 'ALTER TABLE support_tickets ADD (body_vec VECTOR(384, FLOAT32))'; + END IF; +END; +/ + +-- Embed all 10,000 ticket bodies in-database — no external embedding service, no +-- copy pipeline. ~3 min on the Free container (~55 embeddings/s, single-threaded). +-- Re-runnable: recomputes from body. +UPDATE support_tickets SET body_vec = VECTOR_EMBEDDING(MINILM_L12 USING body AS data); + +COMMIT; + +-- IVF (NEIGHBOR PARTITIONS) vector index on the real 384-dim column. IVF needs +-- NO Vector Pool / VECTOR_MEMORY_SIZE, so it builds within the Free container's +-- SGA. Guarded for idempotent re-runs. +DECLARE +BEGIN + EXECUTE IMMEDIATE 'DROP INDEX tickets_bodyvec_ivf'; +EXCEPTION WHEN OTHERS THEN NULL; /* absent — first run */ +END; +/ +CREATE VECTOR INDEX tickets_bodyvec_ivf ON support_tickets(body_vec) + ORGANIZATION NEIGHBOR PARTITIONS + DISTANCE COSINE; + +-- Fresh optimizer statistics on the tables module 03's filtered-ANN plan spans. +-- Load-bearing for determinism: without stats the CBO's dynamic sampling can pick +-- the IVF plan even at toy sizes; with stats the choice is honestly cost-based — +-- full-scan exact distance at hundreds of rows, the IVF index + in-plan filter +-- predicates at 10,000 (the crossover arrives by roughly 3,000 rows on the +-- Free container). +-- (schema named explicitly: this script runs as SYS with CURRENT_SCHEMA=lab_user, +-- and DBMS_STATS resolves USER to the logon user, not the current schema) +BEGIN + DBMS_STATS.GATHER_TABLE_STATS('LAB_USER', 'SUPPORT_TICKETS'); + DBMS_STATS.GATHER_TABLE_STATS('LAB_USER', 'CUSTOMERS'); + DBMS_STATS.GATHER_TABLE_STATS('LAB_USER', 'ORDERS'); +END; +/ diff --git a/labs/converged-database-lab/docker/init/09-hybrid-index.sql b/labs/converged-database-lab/docker/init/09-hybrid-index.sql new file mode 100644 index 0000000..50ed860 --- /dev/null +++ b/labs/converged-database-lab/docker/init/09-hybrid-index.sql @@ -0,0 +1,29 @@ +ALTER SESSION SET CONTAINER = FREEPDB1; +ALTER SESSION SET CURRENT_SCHEMA = lab_user; + +-- Hybrid Vector Index on the ticket body: one domain index unifying Oracle Text +-- (keyword) and a vector index (semantic) over the same column, queried via +-- DBMS_HYBRID_VECTOR.SEARCH with RRF/score fusion (module 03, 04-hybrid-search.sql). +-- Leaf-3-specific: the foundational model + body_vec + IVF live in 08 (on main); +-- this HVI is the article-3 hybrid-search showcase and stays on this branch. +-- +-- COLLISION NOTE — RESOLVED ON MAIN: a CONTEXT/SEARCH text index and the HVI's +-- internal text component are the SAME indextype, so two cannot sit on the same +-- column (ORA-29879). On main, 05-text-vector.sql creates ticket_text_idx on +-- SUBJECT (SYNC ON COMMIT, for module 02's transactional read-after-write +-- proof) and this HVI owns BODY text search (module 03's hybrid proof) — +-- different columns, no collision, so the branch-era ticket_text_idx drop is +-- gone. The HVI's text component does NOT sync on commit (verified 2026-07-24), +-- which is why module 02's proof cannot ride on the HVI. +-- +-- VECTOR_IDXTYPE IVF keeps the HVI's vector component off the Vector Pool +-- (Free-safe); MEMORY 256M caps the Oracle Text build memory modestly. Builds in +-- a few minutes over the 10,000-ticket corpus on the Free container. +DECLARE +BEGIN + EXECUTE IMMEDIATE 'DROP INDEX tickets_hvi'; +EXCEPTION WHEN OTHERS THEN NULL; /* absent — first run */ +END; +/ +CREATE HYBRID VECTOR INDEX tickets_hvi ON support_tickets(body) + PARAMETERS('MODEL MINILM_L12 VECTOR_IDXTYPE IVF MEMORY 256M'); diff --git a/labs/converged-database-lab/modules/02-converged-vs-multi-model/README.md b/labs/converged-database-lab/modules/02-converged-vs-multi-model/README.md index b719139..c632065 100644 --- a/labs/converged-database-lab/modules/02-converged-vs-multi-model/README.md +++ b/labs/converged-database-lab/modules/02-converged-vs-multi-model/README.md @@ -92,7 +92,12 @@ immediately does not. Search visibility arrives with the commit, in the same engine, with no separate search process, change stream, or refresh interval. The lab's `ticket_text_idx` is created with `PARAMETERS ('SYNC (ON COMMIT)')` (`docker/init/05-text-vector.sql`), so the text index syncs inside the -committing transaction. +committing transaction. On main, this index lives on the ticket **subject** +(the marker token travels in the subject accordingly): module 03's Hybrid +Vector Index owns body text search, and an HVI's text component does not sync +on commit — the transactional read-after-write property this proof +demonstrates needs the dedicated SYNC(ON COMMIT) index. The article/02 branch +keeps the body-marker form that the published article's snippets byte-match. **Commit exception:** this script intentionally COMMITs (read-after-write can only be shown across a real commit boundary), so the harness rollback does not diff --git a/labs/converged-database-lab/modules/02-converged-vs-multi-model/scripts/03-read-after-write-search.sql b/labs/converged-database-lab/modules/02-converged-vs-multi-model/scripts/03-read-after-write-search.sql index f5dd8ba..1c6bfa9 100644 --- a/labs/converged-database-lab/modules/02-converged-vs-multi-model/scripts/03-read-after-write-search.sql +++ b/labs/converged-database-lab/modules/02-converged-vs-multi-model/scripts/03-read-after-write-search.sql @@ -7,6 +7,14 @@ DELETE /* Module 02 proof 3: transactional read-after-write text search. syncs inside the committing transaction, so search visibility arrives WITH the commit, not eventually after it. + MAIN-BRANCH RECONCILIATION: on main the SYNC(ON COMMIT) search index lives + on SUBJECT (module 03's Hybrid Vector Index owns body text search, and the + HVI's text component does not sync on commit — verified 2026-07-24), so + this proof carries its marker token in the subject and probes + CONTAINS(subject, ...). The article/02 branch keeps the body-marker form + that the published article's snippets byte-match; the property proven — + search visibility inside the committing transaction — is identical. + COMMIT WARNING: this script intentionally COMMITs — a documented exception to the rollback contract (see the module README). Read-after-write through a text index can only be demonstrated across a real commit boundary. @@ -17,11 +25,11 @@ DELETE /* Module 02 proof 3: transactional read-after-write text search. This first statement is an idempotence guard: remove any probe rows left by a previously interrupted run (committed together with the INSERT below). */ -FROM support_tickets WHERE subject = 'm02 rw-search probe'; +FROM support_tickets WHERE subject LIKE 'm02 rw-search probe%'; INSERT INTO support_tickets (customer_id, subject, body, status) -VALUES (1, 'm02 rw-search probe', - 'read-after-write search probe, marker token zzqxw9347', 'open'); +VALUES (1, 'm02 rw-search probe zzqxw9347', + 'read-after-write search probe (marker token in subject)', 'open'); COMMIT; @@ -29,10 +37,10 @@ SELECT /* the committed row is findable by CONTAINS immediately — same call stack, same second, no sync window */ 'ASSERT:contains-finds-committed-write:' || CASE WHEN COUNT(*) = 1 THEN 'PASS' ELSE 'FAIL' END -FROM support_tickets WHERE CONTAINS(body, 'zzqxw9347') > 0; +FROM support_tickets WHERE CONTAINS(subject, 'zzqxw9347') > 0; DELETE /* explicit cleanup: remove the probe ticket */ -FROM support_tickets WHERE subject = 'm02 rw-search probe'; +FROM support_tickets WHERE subject LIKE 'm02 rw-search probe%'; COMMIT; @@ -40,9 +48,9 @@ SELECT /* and the committed delete is just as immediately invisible to search — read-after-write holds in both directions */ 'ASSERT:contains-after-cleanup:' || CASE WHEN COUNT(*) = 0 THEN 'PASS' ELSE 'FAIL' END -FROM support_tickets WHERE CONTAINS(body, 'zzqxw9347') > 0; +FROM support_tickets WHERE CONTAINS(subject, 'zzqxw9347') > 0; SELECT /* domain restored: no probe rows survive in the base table either */ 'ASSERT:probe-rows-gone:' || CASE WHEN COUNT(*) = 0 THEN 'PASS' ELSE 'FAIL' END -FROM support_tickets WHERE subject = 'm02 rw-search probe'; +FROM support_tickets WHERE subject LIKE 'm02 rw-search probe%'; diff --git a/labs/converged-database-lab/modules/03-vector-vs-converged/README.md b/labs/converged-database-lab/modules/03-vector-vs-converged/README.md new file mode 100644 index 0000000..751e734 --- /dev/null +++ b/labs/converged-database-lab/modules/03-vector-vs-converged/README.md @@ -0,0 +1,322 @@ +# Module 03 — Vector Database vs Converged Database + +Four runnable proofs for leaf article 3 ("Vector Database vs Converged +Database"). A vector database stores embeddings and finds the nearest ones — +one capability production retrieval needs. This module proves the other four: +**filter + join to operational data, read-after-write freshness, +permission-aware retrieval, and hybrid keyword + semantic search** all run +*beside* similarity in one engine, one transaction, one optimizer, one +governance domain. + +Every script executes against the live lab container and emits machine-checkable +assertions (`ASSERT::PASS|FAIL`). No screenshots, no trust-me. + +This is the **deep, native showcase**: the similarity search here runs on **real +384-dimension embeddings over 10,000 support tickets** generated *inside the +database* by Oracle's prebuilt augmented `all-MiniLM-L12-v2` ONNX model, indexed +by both a native IVF vector index and a native **Hybrid Vector Index** — not toy +vectors, and at a size where the cost-based optimizer genuinely chooses the +vector index. See +[Infrastructure](#infrastructure-real-in-db-embeddings--hybrid-vector-index). + +--- + +## Proof 1: `scripts/01-filtered-joined-ann.sql` + +**Article claim (§4): filter + join + similarity, one statement, one plan — with +the vector index in the plan and the predicate list proving filter-first.** A +single SQL statement finds the support tickets most semantically similar to a +natural-language probe (`unable to log in after reset`, embedded in-database by +the same model that embedded the bodies) — but only for `premium`/`vip` +customers with a recently `shipped` order. `EXPLAIN PLAN` + `DBMS_XPLAN` +(`BASIC +PREDICATE`) prove: the **IVF vector index drives the search** (its +`VECTOR$TICKETS_BODYVEC_IVF$…` centroid/partition row sources are in the plan), +`CUSTOMERS`, `ORDERS`, and `SUPPORT_TICKETS` are all costed in **one plan +tree**, and — the predicate-list proof — the segment filter is evaluated **on +the CUSTOMERS scan** and the status/date filter **on the ORDERS scan**, both +*below* the final top-10 `SORT ORDER BY STOPKEY`. The filters run before the +ranking: they are never applied *after* a pre-ranked top-k, which is pgvector's +documented post-index mode. A dedicated store answers only the similarity half — the segment filter +is flat-metadata-only and the join to orders is an application-side fetch +against a second system. + +**Optimizer honesty (scale note).** This is a genuine cost-based choice, not a +hint. With fresh statistics (gathered at init) the same query **full-scans at +300 rows** — exact COSINE over a few hundred rows is cheaper than probing +centroid partitions — and the crossover to the IVF index arrives by roughly +3,000 rows on the Free container; the seeded 10,000 gives the index-driven plan +comfortable margin. +Oracle documents both the pre-filter and post-filter IVF strategies and costs +them; the optimizer weighs, then picks. This module proves **composition and +in-plan filtering, not ANN throughput** — these are correctness/colocation +proofs on a 2-core / Free container, never a latency or recall benchmark. The +final assertion confirms the IVF approximate index genuinely exists and is +`VALID`. + +The captured plan (`DBMS_XPLAN ... 'BASIC +PREDICATE'`, fresh cold build; plan +hash and the `VECTOR$…$` object suffixes vary per build): + +``` +-------------------------------------------------------------------------------------------------------------------- +| Id | Operation | Name | +-------------------------------------------------------------------------------------------------------------------- +| 0 | SELECT STATEMENT | | +|* 1 | COUNT STOPKEY | | +| 2 | VIEW | | +|* 3 | SORT ORDER BY STOPKEY | | +|* 4 | HASH JOIN | | +|* 5 | TABLE ACCESS FULL | ORDERS | +|* 6 | HASH JOIN | | +|* 7 | TABLE ACCESS FULL | CUSTOMERS | +| 8 | NESTED LOOPS | | +| 9 | VIEW | VW_IVENJ_7E289AC7 | +|* 10 | HASH JOIN | | +| 11 | PART JOIN FILTER CREATE | :BF0000 | +| 12 | VIEW | VW_IVCR_B5B87E67 | +|* 13 | COUNT STOPKEY | | +| 14 | VIEW | VW_IVCN_9A1D2119 | +|* 15 | SORT ORDER BY STOPKEY | | +| 16 | TABLE ACCESS FULL | VECTOR$TICKETS_BODYVEC_IVF$...$IVF_FLAT_CENTROIDS | +| 17 | PARTITION LIST JOIN-FILTER| | +| 18 | TABLE ACCESS FULL | VECTOR$TICKETS_BODYVEC_IVF$...$IVF_FLAT_CENTROID_PARTITIONS | +| 19 | TABLE ACCESS BY USER ROWID | SUPPORT_TICKETS | +-------------------------------------------------------------------------------------------------------------------- + +Predicate Information (identified by operation id): +--------------------------------------------------- + 1 - filter(ROWNUM<=10) + 3 - filter(ROWNUM<=10) + 4 - access("O"."CUSTOMER_ID"="C"."CUSTOMER_ID") + 5 - filter("O"."STATUS"='shipped' AND "O"."ORDER_TS">=TIMESTAMP' 2026-03-01 00:00:00.000000000') + 6 - access("C"."CUSTOMER_ID"="T"."CUSTOMER_ID") + 7 - filter("C"."SEGMENT"='premium' OR "C"."SEGMENT"='vip') + 10 - access("VW_IVCR_B5B87E67"."CENTROID_ID"="VTIX_CNPART"."CENTROID_ID") + 13 - filter(ROWNUM<=14) + 15 - filter(ROWNUM<=14) +``` + +Read it bottom-up: the IVF index's top centroids are chosen (ids 12–16, the +`ROWNUM<=14` probe), the matching centroid partitions produce similarity +candidates (17–18), candidates fetch their base rows by rowid (19), the +customer-segment filter runs on the CUSTOMERS scan (id 7) and the order +status/date filter on the ORDERS scan (id 5) — both visible in the predicate +list — and the top-10 `SORT ORDER BY STOPKEY` (id 3) ranks only the filtered, +joined survivors. **Filter first, then rank.** (The candidates are still +accuracy-bounded — only the probed partitions are searched — so this is +approximate search; the point is the filters are inside the plan, not applied +to a pre-ranked top-10.) + +Expected assertions: + +``` +ASSERT:plan-captured:PASS +ASSERT:ivf-index-drives-plan:PASS +ASSERT:plan-spans-customers:PASS +ASSERT:plan-spans-orders:PASS +ASSERT:plan-spans-vector-table:PASS +ASSERT:segment-filter-in-plan:PASS +ASSERT:order-filter-in-plan:PASS +ASSERT:filters-precede-ranking:PASS +ASSERT:vector-drives-sort:PASS +ASSERT:one-plan-tree:PASS +ASSERT:ivf-index-valid:PASS +``` + +## Proof 2: `scripts/02-vector-read-after-write.sql` + +**Article claim (§5): a committed embedding change is immediately visible to +ANN.** The script captures ticket #1's distance and rank to a semantic probe, +`UPDATE`s its `body_vec` to an unrelated embedding, `COMMIT`s, and re-runs the +identical query in the same session — the row's rank collapses (it drops out of +the top-k) *immediately*, with no reindex, no sleep, no LSN poll. It then +recomputes the original embedding from the unchanged `body` text and `COMMIT`s +again; the restored distance is identical to the original. Oracle maintains the +vector index inside the committing transaction (MVCC) — there is no replication +pipeline between models to lag. Contrast (vendor-documented, July 2026): Pinecone +is eventually consistent and requires `x-pinecone-request-lsn` polling; MongoDB +Search/Vector Search (`mongot`) replicates the index off change streams +asynchronously with no read-after-write guarantee. + +**COMMIT exception + restore.** The validator rolls SQL scripts back to leave the +domain unchanged, but this proof must `COMMIT` to show cross-statement visibility, +and `COMMIT` cannot be rolled back. So the proof is fully self-restoring: it +re-embeds the untouched `body` to recover the exact original vector, and the +helper table `m03_raw_proof` (DDL, autocommits) is dropped at the end. On success +the domain is byte-for-byte pristine (asserted). + +Expected assertions: + +``` +ASSERT:committed-write-visible:PASS +ASSERT:rank-changed:PASS +ASSERT:restored-exact:PASS +ASSERT:teardown-clean:PASS +``` + +## Proof 3: `scripts/03-permission-aware-retrieval.sql` + +**Article claim (§6): same vector query, different users, different rows — +governed in-engine.** A `DBMS_RLS` (Virtual Private Database) policy on +`support_tickets`, keyed to an application context, partitions the tickets +between two tenants (A owns even-numbered customers, B owns odd). The **identical** +`FETCH APPROX` vector query is run as each tenant; the result sets are **disjoint** +and each is provably governed by the policy — unauthorized rows never enter the +other tenant's top-k, query unchanged. This is the cleanest expression of the +series' "one governance domain" guarantee: the same row-level policy that governs +SQL, the document API, and graph traversal also governs vector retrieval. +Contrast: Pinecone's own access-control guide concedes the store does not enforce +permissions and must integrate an external authorizer (SpiceDB) and keep ACLs in +sync. (VPD is available in 26ai Free — Licensing guide Table 1-11, Free = Y — +overturning the stale "VPD is Enterprise-only" claim.) + +**Mechanism honesty.** True OS-level multi-user auth is awkward to script as +`LAB_USER`, so the policy predicate keys off an **application context** +(`SYS_CONTEXT`), toggled between two tenant identities via +`DBMS_SESSION.SET_CONTEXT` within one session. The query text is byte-for-byte +identical across both runs; only the governed identity differs — which is the +point. + +**DDL exception + teardown.** The context, context-setter procedure, +policy-predicate function, the `DBMS_RLS` policy, and the helper table are all DDL +(autocommit, not reversible by the validator rollback). Cleanup is explicit and +asserted: all are dropped before the script ends, leaving `support_tickets` +unrestricted with 10,000 rows. + +Expected assertions: + +``` +ASSERT:both-tenants-nonempty:PASS +ASSERT:results-disjoint:PASS +ASSERT:governed-by-policy:PASS +ASSERT:teardown-clean:PASS +``` + +## Proof 4: `scripts/04-hybrid-search.sql` + +**Article claim (§7): one engine fuses keyword + semantic.** The native **Hybrid +Vector Index** (`tickets_hvi`) is a single domain index combining Oracle Text and +a vector index over the ticket body. `DBMS_HYBRID_VECTOR.SEARCH` runs both a +keyword search (`text.contains`) and a semantic search (`vector.search_text`) in +**one call** and returns a single fused, ranked result set — every returned row +carries **both** a `vector_score` and a `text_score`. The proof captures the +fused top-10 and, separately, keyword-only (`CONTAINS`) and vector-only +(`VECTOR_DISTANCE` on `body_vec`) top-10, then asserts the fused set differs from +the keyword-only set (semantic recall surfaces rows lexical ranking misses). +Contrast: pgvector + `pg_search` require hand-rolled reciprocal-rank fusion in +application SQL because BM25 and vector scores are "incommensurable"; Pinecone's +sparse-dense scores are not normalized to the dense range so the app must reweight; +MongoDB's `$rankFusion` is Preview (8.1+). Anthropic's Contextual Retrieval +(Sept 2024) found adding keyword/BM25 to embeddings cut retrieval failures by up +to 67% in their evaluation — hybrid, not raw ANN, is the production stack. + +The exact `DBMS_HYBRID_VECTOR.SEARCH` JSON that runs: + +```json +{ "hybrid_index_name" : "tickets_hvi", + "vector" : { "search_text" : "the box was crushed and broken in transit" }, + "text" : { "contains" : "damaged" }, + "return" : { "topN" : 10 } } +``` + +It returns a JSON array of `{ rowid, score, vector_score, text_score, +vector_rank, text_rank, chunk_text, chunk_id }` rows, unnested with `JSON_TABLE` +and joined back to `support_tickets` by `rowid`. + +**DDL exception.** The helper table `m03_hybrid_proof` autocommits and is dropped +at the end (asserted); the HVI and all 10,000 seeded rows are untouched. + +Expected assertions: + +``` +ASSERT:hybrid-returns-topN:PASS +ASSERT:fusion-uses-both-signals:PASS +ASSERT:fusion-differs-from-keyword:PASS +ASSERT:teardown-clean:PASS +``` + +The module emits **23 assertions across four scripts.** + +--- + +## Infrastructure: real in-DB embeddings + Hybrid Vector Index + +This module adds two pieces of branch-specific infrastructure (built once on +first boot, baked deterministically into the image so CI's fresh build is +reproducible): + +1. **The ONNX model is baked into the image.** `docker/Dockerfile.oracle` + downloads Oracle's official prebuilt augmented `all-MiniLM-L12-v2` ONNX model + (the URL the 26ai docs "ONNX Pipeline Models: Text Embedding" page redirects + to, in the `OML-ai-models` object-storage bucket; ~122.6 MB zip → + ~133.3 MB `.onnx`, 384-dim output) to `/opt/oracle/models`. The build fails + loudly if the URL stops resolving rather than baking a broken image. + +2. **`docker/init/08-vector-model.sql`** (runs after `01`–`07`): loads the model + as `MINILM_L12` via `DBMS_VECTOR.LOAD_ONNX_MODEL`, adds a real + `body_vec VECTOR(384, FLOAT32)` column to `support_tickets`, embeds all + 10,000 ticket bodies in-database with `VECTOR_EMBEDDING(MINILM_L12 USING body + AS data)` (~3 min single-threaded on the Free container), builds an IVF + vector index (`tickets_bodyvec_ivf`, `NEIGHBOR PARTITIONS` / COSINE — no + Vector Pool, Free-safe), and gathers optimizer statistics on + `SUPPORT_TICKETS` / `CUSTOMERS` / `ORDERS` so the index-vs-scan choice in + proof 1 is deterministic. `09-hybrid-index.sql` then builds the Hybrid Vector + Index (`tickets_hvi`, `VECTOR_IDXTYPE IVF MEMORY 256M`) over the same 10,000 + bodies. The corpus itself is seeded by `06-seed.sql`: tickets 1–300 keep the + original 4-template bodies byte-for-byte (module 01's ticket #1 proofs and + proof 2's probe depend on ticket #1), and tickets 301–10,000 are generated + from a deterministic (fixed-LCG) 6×8×4×50×5 phrase grid, so the embeddings + form real semantic clusters and every fresh build is identical. + +**Why IVF, not HNSW.** IVF (`NEIGHBOR PARTITIONS`) needs no Vector Pool / +`VECTOR_MEMORY_SIZE`, so it fits the Free container's SGA. HNSW must live in the +SGA Vector Pool; the module defaults to IVF everywhere to stay Free-safe and does +not require HNSW. + +**The toy `embedding` column stays.** Module 01's deterministic +`VECTOR(8, FLOAT32)` `embedding` column and its IVF index (`ticket_vec_idx`) are +left completely untouched — module 01 still uses them. Module 03 uses the new +`body_vec` and the HVI exclusively. (Module 01's vector assertions — +`txn-vector-visible`, `plan-spans-vector` — continue to pass on the toy column.) + +**Text-index collision (merge note).** A CONTEXT/SEARCH text index and the HVI's +internal text component are the same indextype, so two cannot coexist on the same +column (`ORA-29879`). The shared `05-text-vector.sql` creates `ticket_text_idx` +(a SEARCH index) on `body`; module 03 needs the richer body text for a meaningful +hybrid proof, so `08-vector-model.sql` drops `ticket_text_idx` and lets the HVI +own body's text search (the HVI exposes the same `CONTAINS(body, ...)` +capability, which proof 4's keyword-only path uses). On the eventual merge with +the module-02 branch, whose read-after-write SEARCH proof uses `ticket_text_idx`, +that proof moves to the HVI or to the `subject` column — a merge-time decision. + +--- + +## Run It Yourself + +```bash +docker compose up -d --build oracle # first build downloads the ~122 MB model +pip install -r validator/requirements.txt +python validator/run.py +``` + +On hosts where docker is not permitted (e.g. Oracle-managed laptops), podman +works end to end — note docker.io pulls may fail behind corporate TLS +interception; pull the base image from `ghcr.io` and retag first: + +```bash +podman pull ghcr.io/gvenzl/oracle-free:23.26.0-faststart +podman tag ghcr.io/gvenzl/oracle-free:23.26.0-faststart docker.io/gvenzl/oracle-free:23.26.0-faststart +podman build --pull=never -f docker/Dockerfile.oracle -t cdl-oracle docker/ +podman run -d --name lab-oracle -p 1521:1521 -p 8181:8181 -p 27017:27017 \ + -e ORACLE_PASSWORD=LabAdmin2026 -e APP_USER=LAB_USER -e APP_USER_PASSWORD=LabUser2026 cdl-oracle +CONTAINER_ENGINE=podman python validator/run.py +``` + +First boot embeds 10,000 ticket bodies in-database and builds the HVI over them, +so init takes ~8–10 minutes; the compose healthcheck allows for it. + +The validator runs every script and exits 0 only if all assertions pass. + +**These proofs leave the seeded domain unchanged.** Proofs that must `COMMIT` +(read-after-write) or perform DDL (the VPD policy, the helper tables) restore or +drop everything they touch and assert the teardown, so the validator runs any +number of times with identical results. diff --git a/labs/converged-database-lab/modules/03-vector-vs-converged/scripts/01-filtered-joined-ann.sql b/labs/converged-database-lab/modules/03-vector-vs-converged/scripts/01-filtered-joined-ann.sql new file mode 100644 index 0000000..17581ad --- /dev/null +++ b/labs/converged-database-lab/modules/03-vector-vs-converged/scripts/01-filtered-joined-ann.sql @@ -0,0 +1,175 @@ +DELETE /* Module 03 proof 1: similarity AND relational filter AND join, in ONE + optimizer-costed plan — with the IVF vector index driving the search and the + relational predicates PROVABLY applied inside the plan, before the ranking. + Against REAL 384-dim embeddings over 10,000 tickets. + + The query is a production-shaped RAG retrieval: find the support tickets most + semantically similar to a natural-language probe ("unable to log in after + reset"), but only for premium/vip customers who have a recently SHIPPED order + — similarity composed with live business predicates and a join to operational + tables. The probe is embedded in-database by the same model that embedded the + ticket bodies (VECTOR_EMBEDDING(MINILM_L12 USING ... AS data)), so it is a + real semantic probe, not a canned literal vector. + + A dedicated vector store answers only the similarity half: the segment filter + is flat-metadata-only, and the join to ORDERS is an application-side fetch + against a second system. Here CUSTOMERS, ORDERS, and SUPPORT_TICKETS are all + costed in ONE plan tree by ONE cost-based optimizer — and the DBMS_XPLAN + predicate list shows WHERE each filter runs. + + WHAT THE PLAN SHOWS (the filter-first proof): at 10,000 tickets the CBO + chooses the IVF vector index (VECTOR$TICKETS_BODYVEC_IVF$... row sources): + the index's top centroid partitions produce the similarity candidates, the + candidates join back to SUPPORT_TICKETS by rowid, the segment predicate is + evaluated on the CUSTOMERS scan and the status/date predicate on the ORDERS + scan — all VISIBLE in the Predicate Information section — and only the + filtered, joined survivors reach the final top-10 SORT ORDER BY STOPKEY. + The filters run before the ranking — they are never applied AFTER a + pre-ranked top-10, which is pgvector's documented post-index mode (a + 10%-selective predicate leaving ~4 rows of a requested 10). + + OPTIMIZER HONESTY: this is a genuine cost-based choice, not a hint. With + fresh stats (gathered at init) the same query full-scans at 300 rows — + exact distance over a few hundred rows is simply cheaper than probing + centroid partitions — and the crossover to the IVF index arrives by + roughly 3,000 rows on the Free container. Oracle documents both the pre-filter and post-filter IVF + strategies and costs them; the optimizer weighs, then picks. This module + proves COMPOSITION and in-plan filtering on a 2-core Free container — it is + never a latency or recall benchmark (see README scale note). This first + statement is an idempotence guard. */ +FROM plan_table WHERE statement_id = 'm03-filtered'; + +EXPLAIN PLAN SET STATEMENT_ID = 'm03-filtered' FOR +SELECT t.ticket_id, + c.full_name /* non-key column keeps CUSTOMERS in the plan (defeats join + elimination) so the relational join is visibly costed */, + o.order_id, + VECTOR_DISTANCE(t.body_vec, + VECTOR_EMBEDDING(MINILM_L12 USING 'unable to log in after reset' AS data), + COSINE) AS dist +FROM support_tickets t +JOIN customers c ON c.customer_id = t.customer_id +JOIN orders o ON o.customer_id = c.customer_id +WHERE c.segment IN ('premium','vip') + AND o.status = 'shipped' + AND o.order_ts >= TIMESTAMP '2026-03-01 00:00:00' +ORDER BY dist +FETCH APPROX FIRST 10 ROWS ONLY WITH TARGET ACCURACY 90; + +SELECT /* the human-readable plan WITH the predicate list — the exhibit the + article prints. Not an assertion; the assertions below check the same + facts machine-readably from PLAN_TABLE. */ + plan_table_output +FROM TABLE(DBMS_XPLAN.DISPLAY('PLAN_TABLE', 'm03-filtered', 'BASIC +PREDICATE')); + +SELECT /* a real costed plan was produced (>= 12 row sources: 3 domain tables, + the IVF index row sources, joins, sorts, stopkeys) */ + 'ASSERT:plan-captured:' || + CASE WHEN COUNT(*) >= 12 THEN 'PASS' ELSE 'FAIL' END +FROM plan_table WHERE statement_id = 'm03-filtered'; + +SELECT /* the IVF vector index drives the plan — its centroid/partition row + sources (VECTOR$TICKETS_BODYVEC_IVF$...) are in the plan tree, so the + similarity candidates come from the ANN index, not a full scan */ + 'ASSERT:ivf-index-drives-plan:' || + CASE WHEN EXISTS (SELECT 1 FROM plan_table + WHERE statement_id = 'm03-filtered' + AND object_name LIKE 'VECTOR$TICKETS_BODYVEC_IVF$%') + THEN 'PASS' ELSE 'FAIL' END +FROM dual; + +SELECT /* the operational customer table is costed in the same plan as the + vector search — the filter+join the dedicated store cannot do */ + 'ASSERT:plan-spans-customers:' || + CASE WHEN EXISTS (SELECT 1 FROM plan_table + WHERE statement_id = 'm03-filtered' + AND object_name = 'CUSTOMERS') + THEN 'PASS' ELSE 'FAIL' END +FROM dual; + +SELECT /* the orders table — the operational join target — is in the same plan */ + 'ASSERT:plan-spans-orders:' || + CASE WHEN EXISTS (SELECT 1 FROM plan_table + WHERE statement_id = 'm03-filtered' + AND object_name = 'ORDERS') + THEN 'PASS' ELSE 'FAIL' END +FROM dual; + +SELECT /* the base vector table appears too — the ANN candidates are joined back + to SUPPORT_TICKETS rows (rowid join) inside the same plan tree */ + 'ASSERT:plan-spans-vector-table:' || + CASE WHEN EXISTS (SELECT 1 FROM plan_table + WHERE statement_id = 'm03-filtered' + AND object_name = 'SUPPORT_TICKETS') + THEN 'PASS' ELSE 'FAIL' END +FROM dual; + +SELECT /* the PREDICATE LIST proof, part 1: the segment filter is evaluated ON + the CUSTOMERS row source — inside the plan, not after the ranking */ + 'ASSERT:segment-filter-in-plan:' || + CASE WHEN EXISTS (SELECT 1 FROM plan_table + WHERE statement_id = 'm03-filtered' + AND object_name = 'CUSTOMERS' + AND filter_predicates LIKE '%SEGMENT%') + THEN 'PASS' ELSE 'FAIL' END +FROM dual; + +SELECT /* the PREDICATE LIST proof, part 2: the order-status/date filter is + evaluated ON the ORDERS row source, same plan, before the ranking */ + 'ASSERT:order-filter-in-plan:' || + CASE WHEN EXISTS (SELECT 1 FROM plan_table + WHERE statement_id = 'm03-filtered' + AND object_name = 'ORDERS' + AND filter_predicates LIKE '%STATUS%' + AND filter_predicates LIKE '%ORDER_TS%') + THEN 'PASS' ELSE 'FAIL' END +FROM dual; + +SELECT /* the PREDICATE LIST proof, part 3 — FILTER BEFORE RANK: both relational + filters sit BELOW the final top-10 SORT ORDER BY STOPKEY in the plan + tree (preorder ids: descendants of the ranking sort have larger ids), + so the top-10 is computed OVER the filtered rows — filters shrink the + candidate pool, they are never applied to a pre-ranked top-10 */ + 'ASSERT:filters-precede-ranking:' || + CASE WHEN (SELECT MIN(id) FROM plan_table + WHERE statement_id = 'm03-filtered' + AND operation = 'SORT' AND options LIKE 'ORDER BY%STOPKEY') + < (SELECT MIN(id) FROM plan_table + WHERE statement_id = 'm03-filtered' + AND object_name = 'CUSTOMERS' + AND filter_predicates LIKE '%SEGMENT%') + AND (SELECT MIN(id) FROM plan_table + WHERE statement_id = 'm03-filtered' + AND operation = 'SORT' AND options LIKE 'ORDER BY%STOPKEY') + < (SELECT MIN(id) FROM plan_table + WHERE statement_id = 'm03-filtered' + AND object_name = 'ORDERS' + AND filter_predicates LIKE '%STATUS%') + THEN 'PASS' ELSE 'FAIL' END +FROM dual; + +SELECT /* the VECTOR_DISTANCE drives the ranking — the top-k SORT ORDER BY + (STOPKEY) row source is present, i.e. similarity ordering is in the + same plan as the filters and joins */ + 'ASSERT:vector-drives-sort:' || + CASE WHEN EXISTS (SELECT 1 FROM plan_table + WHERE statement_id = 'm03-filtered' + AND operation = 'SORT' + AND options LIKE 'ORDER BY%') + THEN 'PASS' ELSE 'FAIL' END +FROM dual; + +SELECT /* every row source above belongs to ONE plan tree — one optimizer, one + statement, no federation seam between similarity and the relational join */ + 'ASSERT:one-plan-tree:' || + CASE WHEN COUNT(DISTINCT plan_id) = 1 THEN 'PASS' ELSE 'FAIL' END +FROM plan_table WHERE statement_id = 'm03-filtered'; + +SELECT /* the IVF approximate index on the real 384-dim column is built and + VALID — the same index the plan assertions above show in use */ + 'ASSERT:ivf-index-valid:' || + CASE WHEN COUNT(*) = 1 THEN 'PASS' ELSE 'FAIL' END +FROM user_indexes +WHERE index_name = 'TICKETS_BODYVEC_IVF' + AND index_type = 'VECTOR' + AND status = 'VALID'; diff --git a/labs/converged-database-lab/modules/03-vector-vs-converged/scripts/02-vector-read-after-write.sql b/labs/converged-database-lab/modules/03-vector-vs-converged/scripts/02-vector-read-after-write.sql new file mode 100644 index 0000000..6144957 --- /dev/null +++ b/labs/converged-database-lab/modules/03-vector-vs-converged/scripts/02-vector-read-after-write.sql @@ -0,0 +1,116 @@ +DECLARE /* Module 03 proof 2: a committed embedding change is IMMEDIATELY visible + to the next approximate-similarity query — read-after-write on the vector path, + with no reindex, no sleep, no LSN poll. + + A dedicated/synced vector store is eventually consistent here. Pinecone's own + docs: "there can be a slight delay before new or changed records are visible + to queries," and read-after-write must be hand-built by polling the + x-pinecone-request-lsn / x-pinecone-max-indexed-lsn headers. MongoDB Atlas + ($vectorSearch via mongot) replicates the index off the oplog asynchronously + and gives no read-after-write guarantee. Oracle maintains the vector index + inside the committing transaction (MVCC), so the change is visible on the next + read by any session — there is no replication pipeline between models to lag. + + COMMIT / TEARDOWN EXCEPTION (honest): the validator rolls back SQL scripts to + leave the domain unchanged, but this proof must COMMIT to demonstrate + cross-statement visibility — and COMMIT cannot be rolled back. So the proof is + fully self-restoring: it updates ticket #1's body_vec to an unrelated + embedding, COMMITs, re-measures, then RECOMPUTES the original embedding from + the unchanged `body` text and COMMITs again. The DDL helper table m03_raw_proof + also autocommits and is dropped at the end. On success the domain is byte-for- + byte pristine (restored distance == original distance, asserted). + + This first guard drops any helper table left by an interrupted run. */ +BEGIN + EXECUTE IMMEDIATE 'DROP TABLE m03_raw_proof PURGE'; +EXCEPTION WHEN OTHERS THEN NULL; +END; +/ + +CREATE TABLE m03_raw_proof (k VARCHAR2(30) PRIMARY KEY, v NUMBER); + +DECLARE /* the whole capture / update+commit / re-measure / restore+commit cycle, + in one block so the committed states are observed in order. Results land in + m03_raw_proof for the standalone ASSERT selects below. body_vec is restored by + re-embedding the untouched `body`, so the original value returns exactly. */ + v_qv VECTOR; + v_before_d NUMBER; + v_after_d NUMBER; + v_restored_d NUMBER; + v_before_rank NUMBER; + v_after_rank NUMBER; +BEGIN + /* probe: a paraphrase, not a copy, of ticket #1's "Login fails ... after + reset" body — the match is semantic, not verbatim */ + SELECT VECTOR_EMBEDDING(MINILM_L12 USING 'cannot access my account login error' AS data) + INTO v_qv FROM dual; + + SELECT VECTOR_DISTANCE(body_vec, v_qv, COSINE) INTO v_before_d + FROM support_tickets WHERE ticket_id = 1; + SELECT rnk INTO v_before_rank FROM ( + SELECT ticket_id, ROW_NUMBER() OVER (ORDER BY VECTOR_DISTANCE(body_vec, v_qv, COSINE)) rnk + FROM support_tickets) WHERE ticket_id = 1; + + /* overwrite ticket #1's embedding with something semantically unrelated, COMMIT */ + UPDATE support_tickets + SET body_vec = VECTOR_EMBEDDING(MINILM_L12 USING 'tropical fish aquarium maintenance schedule' AS data) + WHERE ticket_id = 1; + COMMIT; + + /* the very next query sees the new vector immediately — no reindex/sleep */ + SELECT VECTOR_DISTANCE(body_vec, v_qv, COSINE) INTO v_after_d + FROM support_tickets WHERE ticket_id = 1; + SELECT rnk INTO v_after_rank FROM ( + SELECT ticket_id, ROW_NUMBER() OVER (ORDER BY VECTOR_DISTANCE(body_vec, v_qv, COSINE)) rnk + FROM support_tickets) WHERE ticket_id = 1; + + /* restore the original embedding from the unchanged body text, COMMIT */ + UPDATE support_tickets + SET body_vec = VECTOR_EMBEDDING(MINILM_L12 USING body AS data) + WHERE ticket_id = 1; + COMMIT; + + SELECT VECTOR_DISTANCE(body_vec, v_qv, COSINE) INTO v_restored_d + FROM support_tickets WHERE ticket_id = 1; + + INSERT INTO m03_raw_proof VALUES ('before_rank', v_before_rank); + INSERT INTO m03_raw_proof VALUES ('after_rank', v_after_rank); + INSERT INTO m03_raw_proof VALUES ('dist_changed', CASE WHEN ABS(v_after_d - v_before_d) > 0.01 THEN 1 ELSE 0 END); + INSERT INTO m03_raw_proof VALUES ('restore_match', CASE WHEN ABS(v_restored_d - v_before_d) < 0.00001 THEN 1 ELSE 0 END); + COMMIT; +END; +/ + +SELECT /* the committed embedding change moved the row distance to the probe — + the write was immediately visible to the similarity query */ + 'ASSERT:committed-write-visible:' || + CASE WHEN (SELECT v FROM m03_raw_proof WHERE k = 'dist_changed') = 1 + THEN 'PASS' ELSE 'FAIL' END +FROM dual; + +SELECT /* the row ANN rank changed too (it fell far down the top-k after the + unrelated overwrite) — ranking reflects the new vector at once */ + 'ASSERT:rank-changed:' || + CASE WHEN (SELECT v FROM m03_raw_proof WHERE k = 'after_rank') + > (SELECT v FROM m03_raw_proof WHERE k = 'before_rank') + THEN 'PASS' ELSE 'FAIL' END +FROM dual; + +SELECT /* after re-embedding the untouched body and committing, the distance is + identical to the original — the domain is restored byte-for-byte */ + 'ASSERT:restored-exact:' || + CASE WHEN (SELECT v FROM m03_raw_proof WHERE k = 'restore_match') = 1 + THEN 'PASS' ELSE 'FAIL' END +FROM dual; + +DECLARE /* teardown: drop the helper table (DDL autocommits) so the domain is + left exactly as seeded — support_tickets unchanged, no module-local objects */ +BEGIN + EXECUTE IMMEDIATE 'DROP TABLE m03_raw_proof PURGE'; +END; +/ + +SELECT /* the helper table is gone — teardown verified */ + 'ASSERT:teardown-clean:' || + CASE WHEN COUNT(*) = 0 THEN 'PASS' ELSE 'FAIL' END +FROM user_tables WHERE table_name = 'M03_RAW_PROOF'; diff --git a/labs/converged-database-lab/modules/03-vector-vs-converged/scripts/03-permission-aware-retrieval.sql b/labs/converged-database-lab/modules/03-vector-vs-converged/scripts/03-permission-aware-retrieval.sql new file mode 100644 index 0000000..2a938c8 --- /dev/null +++ b/labs/converged-database-lab/modules/03-vector-vs-converged/scripts/03-permission-aware-retrieval.sql @@ -0,0 +1,144 @@ +DECLARE /* Module 03 proof 3: the SAME vector query, run by two tenants, returns + DIFFERENT governed rows — permission-aware retrieval enforced by the engine, + query unchanged. This is the cleanest expression of the series' "one + governance domain" guarantee: the row-level policy that governs SQL, the + document API, and graph traversal also governs vector retrieval, because it is + one engine over one set of rows. + + The documented contrast: permission-aware RAG is a named production problem — + Pinecone's OWN access-control guide concedes the store does not enforce + permissions and must integrate an external authorizer (e.g. SpiceDB), copy ids + into metadata, and keep ACLs in sync with the source system. The permission + model lives OUTSIDE the retrieval engine, a consistency gap by construction. + Here it is a Virtual Private Database (DBMS_RLS) policy — available in 26ai + Free (Licensing guide Table 1-11, Free column = Y) — that ANDs a predicate + into the same FETCH APPROX query the app already issues. + + MECHANISM (honest): true OS-level multi-user auth is awkward to script as + LAB_USER, so we use a VPD policy whose predicate keys off an APPLICATION + CONTEXT, and toggle that context between two tenant identities ('A','B') + within one session via DBMS_SESSION.SET_CONTEXT. The query is byte-for-byte + identical across both; only the governed identity differs — which is exactly + the point. Tenant A owns even-numbered customers' tickets, tenant B owns the + odd-numbered ones; the sets are disjoint and partition all 10,000 tickets. + + DDL / TEARDOWN EXCEPTION: this proof creates a context, a context-setter + procedure, a policy-predicate function, and a DBMS_RLS policy — all DDL, all + autocommitting, none reversible by the validator's rollback. Cleanup is + therefore explicit and asserted: the policy, function, procedure, context, and + the helper table are all dropped before the script ends. This first block + removes any leftovers from an interrupted run. */ +BEGIN + BEGIN DBMS_RLS.DROP_POLICY('LAB_USER','SUPPORT_TICKETS','M03_TENANT_POL'); EXCEPTION WHEN OTHERS THEN NULL; END; + BEGIN EXECUTE IMMEDIATE 'DROP FUNCTION m03_tenant_pred'; EXCEPTION WHEN OTHERS THEN NULL; END; + BEGIN EXECUTE IMMEDIATE 'DROP PROCEDURE m03_set_tenant'; EXCEPTION WHEN OTHERS THEN NULL; END; + BEGIN EXECUTE IMMEDIATE 'DROP CONTEXT m03_tenant_ctx'; EXCEPTION WHEN OTHERS THEN NULL; END; + BEGIN EXECUTE IMMEDIATE 'DROP TABLE m03_perm_proof PURGE';EXCEPTION WHEN OTHERS THEN NULL; END; +END; +/ + +CREATE CONTEXT m03_tenant_ctx USING m03_set_tenant; + +CREATE PROCEDURE m03_set_tenant(p_tenant IN VARCHAR2) IS +BEGIN + DBMS_SESSION.SET_CONTEXT('M03_TENANT_CTX','TENANT', p_tenant); +END; +/ + +CREATE FUNCTION m03_tenant_pred(p_schema VARCHAR2, p_object VARCHAR2) RETURN VARCHAR2 IS +BEGIN /* the row-level predicate: a ticket is visible only if its customer's id + parity matches the current tenant (A=even, B=odd). Folded into every SELECT on + support_tickets, including the vector ORDER BY ... FETCH APPROX query. */ + RETURN q'[customer_id IN (SELECT customer_id FROM customers + WHERE MOD(customer_id,2) = + CASE WHEN SYS_CONTEXT('M03_TENANT_CTX','TENANT')='A' THEN 0 ELSE 1 END)]'; +END; +/ + +BEGIN /* attach the VPD policy to support_tickets for SELECT — the engine now + enforces the predicate on every read, no matter how the rows are retrieved */ + DBMS_RLS.ADD_POLICY( + object_schema => 'LAB_USER', + object_name => 'SUPPORT_TICKETS', + policy_name => 'M03_TENANT_POL', + function_schema => 'LAB_USER', + policy_function => 'M03_TENANT_PRED', + statement_types => 'SELECT'); +END; +/ + +CREATE TABLE m03_perm_proof (tenant VARCHAR2(1), ticket_id NUMBER, PRIMARY KEY (tenant, ticket_id)); + +DECLARE /* run the IDENTICAL approximate vector query as tenant A, then tenant B + and capture each governed top-10 into the helper table. The only thing that + changes between the two runs is the application context — the SQL text is the + same. */ + PROCEDURE capture(p_tenant VARCHAR2) IS + BEGIN + m03_set_tenant(p_tenant); + INSERT INTO m03_perm_proof (tenant, ticket_id) + SELECT p_tenant, ticket_id FROM ( + SELECT ticket_id + FROM support_tickets + ORDER BY VECTOR_DISTANCE(body_vec, + VECTOR_EMBEDDING(MINILM_L12 USING 'refund for a damaged package' AS data), + COSINE) + FETCH APPROX FIRST 10 ROWS ONLY WITH TARGET ACCURACY 90); + END; +BEGIN + capture('A'); + capture('B'); + COMMIT; +END; +/ + +SELECT /* both tenants got a non-empty governed result set from the same query */ + 'ASSERT:both-tenants-nonempty:' || + CASE WHEN (SELECT COUNT(*) FROM m03_perm_proof WHERE tenant='A') > 0 + AND (SELECT COUNT(*) FROM m03_perm_proof WHERE tenant='B') > 0 + THEN 'PASS' ELSE 'FAIL' END +FROM dual; + +SELECT /* the two result sets are DISJOINT — no ticket visible to both tenants, + unauthorized rows never entered the other tenant top-k */ + 'ASSERT:results-disjoint:' || + CASE WHEN NOT EXISTS ( + SELECT ticket_id FROM m03_perm_proof WHERE tenant='A' + INTERSECT + SELECT ticket_id FROM m03_perm_proof WHERE tenant='B') + THEN 'PASS' ELSE 'FAIL' END +FROM dual; + +SELECT /* tenant A rows really are even-customer tickets and tenant B rows really + are odd-customer tickets — the governance predicate, not chance, split them */ + 'ASSERT:governed-by-policy:' || + CASE WHEN (SELECT COUNT(*) FROM m03_perm_proof p JOIN support_tickets t ON t.ticket_id=p.ticket_id + WHERE p.tenant='A' AND MOD(t.customer_id,2) <> 0) = 0 + AND (SELECT COUNT(*) FROM m03_perm_proof p JOIN support_tickets t ON t.ticket_id=p.ticket_id + WHERE p.tenant='B' AND MOD(t.customer_id,2) <> 1) = 0 + THEN 'PASS' ELSE 'FAIL' END +FROM dual; + +DECLARE /* teardown: drop policy, predicate function, setter, context, helper + table — all of it autocommitted DDL, so it must be explicit. The rollback the + validator runs afterward has nothing of ours left to undo. */ +BEGIN + DBMS_RLS.DROP_POLICY('LAB_USER','SUPPORT_TICKETS','M03_TENANT_POL'); + EXECUTE IMMEDIATE 'DROP FUNCTION m03_tenant_pred'; + EXECUTE IMMEDIATE 'DROP PROCEDURE m03_set_tenant'; + EXECUTE IMMEDIATE 'DROP CONTEXT m03_tenant_ctx'; + EXECUTE IMMEDIATE 'DROP TABLE m03_perm_proof PURGE'; +END; +/ + +SELECT /* nothing module-local survives: no policy, no predicate function, no + setter procedure, no helper table — and support_tickets is unrestricted + again (the context namespace, dropped above, is inert without its + predicate function and is not visible in any lab_user dictionary view) */ + 'ASSERT:teardown-clean:' || + CASE WHEN (SELECT COUNT(*) FROM user_policies WHERE policy_name='M03_TENANT_POL') = 0 + AND (SELECT COUNT(*) FROM user_objects + WHERE object_name IN ('M03_TENANT_PRED','M03_SET_TENANT','M03_PERM_PROOF')) = 0 + AND (SELECT COUNT(*) FROM support_tickets) = 10000 + THEN 'PASS' ELSE 'FAIL' END +FROM dual; diff --git a/labs/converged-database-lab/modules/03-vector-vs-converged/scripts/04-hybrid-search.sql b/labs/converged-database-lab/modules/03-vector-vs-converged/scripts/04-hybrid-search.sql new file mode 100644 index 0000000..97ea6dc --- /dev/null +++ b/labs/converged-database-lab/modules/03-vector-vs-converged/scripts/04-hybrid-search.sql @@ -0,0 +1,125 @@ +DECLARE /* Module 03 proof 4: ONE engine, ONE index, ONE call fuses keyword and + semantic search — Oracle's native Hybrid Vector Index (HVI) queried through + DBMS_HYBRID_VECTOR.SEARCH. The HVI (tickets_hvi, built in 08-vector-model.sql) + is a single domain index combining Oracle Text and a vector index over the + same ticket body, with embeddings produced in-database by the all-MiniLM-L12-v2 + ONNX model. + + Why hybrid: Anthropic's Contextual Retrieval (Sept 2024) found that adding + keyword/BM25 to embeddings reduced retrieval failures by up to 67% in their + evaluation — lexical search catches exact ids/codes/acronyms that dense + vectors miss, while vectors catch paraphrase a keyword query misses. The + production stack is hybrid, not raw ANN. + + The contrast: pgvector + pg_search require hand-rolled reciprocal-rank fusion + in application SQL because BM25 and vector scores are "incommensurable" + (ParadeDB's own framing); Pinecone's sparse-dense scores "are not normalized + to the dense vector range," so the app must weight and rerank; MongoDB's + $rankFusion is Preview (8.1+). Here the fusion is one call: the HVI computes + BOTH a vector_score and a text_score per row and returns a single fused, + ranked result set. + + This proof: run the hybrid search (semantic probe "the box was crushed and + broken in transit" + keyword "damaged"), capture the fused top-10; separately + capture keyword-only (CONTAINS) and vector-only (VECTOR_DISTANCE on body_vec) + top-10. Assertions prove (a) the hybrid call returns topN rows, (b) every + fused row carries BOTH a vector and a text score — proof the single index ran + keyword AND semantic together, and (c) the fused set differs from the + keyword-only set — semantic recall surfaces rows lexical ranking alone misses. + + DDL EXCEPTION: the helper table m03_hybrid_proof is DDL (autocommits); it is + dropped at the end and the drop is asserted. The HVI and the seeded rows are + never modified. This first guard clears any leftover helper table. */ +BEGIN + EXECUTE IMMEDIATE 'DROP TABLE m03_hybrid_proof PURGE'; +EXCEPTION WHEN OTHERS THEN NULL; +END; +/ + +CREATE TABLE m03_hybrid_proof ( + method VARCHAR2(8), + ticket_id NUMBER, + vscore NUMBER, + tscore NUMBER, + PRIMARY KEY (method, ticket_id) +); + +DECLARE /* run the three retrievals and stash their top-10 into the helper table. + The hybrid result is the JSON array returned by DBMS_HYBRID_VECTOR.SEARCH, + unnested with JSON_TABLE; rowid maps back to ticket_id. Keyword-only uses the + HVI's text component via CONTAINS; vector-only uses the real 384-dim body_vec + IVF column. All three share the same probe text so the comparison is fair. */ + c_probe CONSTANT VARCHAR2(80) := 'the box was crushed and broken in transit'; + c_keyword CONSTANT VARCHAR2(40) := 'damaged'; + v_json CLOB; +BEGIN + /* hybrid: one call, fused keyword + semantic over the single HVI */ + v_json := DBMS_HYBRID_VECTOR.SEARCH( + json('{ "hybrid_index_name" : "tickets_hvi", + "vector" : { "search_text" : "the box was crushed and broken in transit" }, + "text" : { "contains" : "damaged" }, + "return" : { "topN" : 10 } }')); + INSERT INTO m03_hybrid_proof (method, ticket_id, vscore, tscore) + SELECT 'HYBRID', t.ticket_id, h.vscore, h.tscore + FROM JSON_TABLE(v_json, '$[*]' + COLUMNS (rid VARCHAR2(30) PATH '$.rowid', + vscore NUMBER PATH '$.vector_score', + tscore NUMBER PATH '$.text_score')) h + JOIN support_tickets t ON t.rowid = CHARTOROWID(h.rid); + + /* keyword-only: pure lexical, the HVI text side via CONTAINS */ + INSERT INTO m03_hybrid_proof (method, ticket_id) + SELECT 'KEYWORD', ticket_id FROM ( + SELECT ticket_id FROM support_tickets + WHERE CONTAINS(body, c_keyword, 1) > 0 + ORDER BY SCORE(1) DESC, ticket_id + FETCH FIRST 10 ROWS ONLY); + + /* vector-only: pure semantic, VECTOR_DISTANCE on the real body_vec column */ + INSERT INTO m03_hybrid_proof (method, ticket_id) + SELECT 'VECTOR', ticket_id FROM ( + SELECT ticket_id FROM support_tickets + ORDER BY VECTOR_DISTANCE(body_vec, + VECTOR_EMBEDDING(MINILM_L12 USING c_probe AS data), COSINE) + FETCH FIRST 10 ROWS ONLY); + COMMIT; +END; +/ + +SELECT /* the hybrid call returned the requested top-N (10) fused rows */ + 'ASSERT:hybrid-returns-topN:' || + CASE WHEN (SELECT COUNT(*) FROM m03_hybrid_proof WHERE method='HYBRID') = 10 + THEN 'PASS' ELSE 'FAIL' END +FROM dual; + +SELECT /* every fused row carries BOTH a vector score and a text score — proof + the single HVI ran keyword AND semantic search together and fused them + in one call, no app-side RRF across two systems, no score normalization */ + 'ASSERT:fusion-uses-both-signals:' || + CASE WHEN (SELECT COUNT(*) FROM m03_hybrid_proof + WHERE method='HYBRID' AND vscore IS NOT NULL AND tscore IS NOT NULL) = 10 + THEN 'PASS' ELSE 'FAIL' END +FROM dual; + +SELECT /* the fused result set differs from the keyword-only set — semantic + recall pulls in rows pure lexical ranking does not surface (the + hybrid-vs-lexical gap that Anthropic measured) */ + 'ASSERT:fusion-differs-from-keyword:' || + CASE WHEN EXISTS ( + SELECT ticket_id FROM m03_hybrid_proof WHERE method='HYBRID' + MINUS + SELECT ticket_id FROM m03_hybrid_proof WHERE method='KEYWORD') + THEN 'PASS' ELSE 'FAIL' END +FROM dual; + +DECLARE /* teardown: drop the helper table (DDL autocommits). The HVI and all 300 + seeded ticket rows are untouched by this proof. */ +BEGIN + EXECUTE IMMEDIATE 'DROP TABLE m03_hybrid_proof PURGE'; +END; +/ + +SELECT /* helper table gone — teardown verified, domain unchanged */ + 'ASSERT:teardown-clean:' || + CASE WHEN COUNT(*) = 0 THEN 'PASS' ELSE 'FAIL' END +FROM user_tables WHERE table_name = 'M03_HYBRID_PROOF'; diff --git a/labs/converged-database-lab/validator/run.py b/labs/converged-database-lab/validator/run.py index 5ac569a..9b1e511 100644 --- a/labs/converged-database-lab/validator/run.py +++ b/labs/converged-database-lab/validator/run.py @@ -76,7 +76,10 @@ def run_sql(path, conn): return asserts, errors def run_js(path): - cmd = ["docker", "exec", "-i", "lab-oracle", "mongosh", "--quiet", MONGO_URI, + # CONTAINER_ENGINE=podman for hosts where docker is not permitted (e.g. Oracle + # laptops); CI keeps the docker default. + engine = os.environ.get("CONTAINER_ENGINE", "docker") + cmd = [engine, "exec", "-i", "lab-oracle", "mongosh", "--quiet", MONGO_URI, "--file", "/dev/stdin"] with open(path, "rb") as fh: proc = subprocess.run(cmd, stdin=fh, capture_output=True, text=True, timeout=300)