Merge article/03-vector-vs-converged: module 03 lab (10K corpus, IVF filter-first proof, HVI)

# Conflicts:
#	labs/converged-database-lab/docker/init/08-vector-model.sql

Author: rhoulihan

labs/converged-database-lab/.gitignore

@@ -5,3 +5,4 @@ validator/results.json
 validator/.venv/
 docker/.data/
 *.pyc
+tasks/

labs/converged-database-lab/README.md

@@ -1,6 +1,6 @@
 # converged-database-lab
 
-![validate](https://github.com/oracle-devrel/oracle-umt-developer-hub/actions/workflows/converged-database-lab.yml/badge.svg)
+![validate](https://github.com/rhoulihan/converged-database-lab/actions/workflows/validate.yml/badge.svg)
 
 Runnable proofs for the converged-database article series — every technical claim in the articles executes here, nightly, against a free container.
 

labs/converged-database-lab/docker-compose.yml

@@ -17,7 +17,10 @@ services:
       test: ['CMD-SHELL', '/opt/oracle/healthcheck.sh && curl -so /dev/null -w "%{http_code}" http://localhost:8181/ | grep -q "302\|200"']
       interval: 15s
       timeout: 10s
-      retries: 40
+      # 80 retries: first boot now embeds 10,000 ticket bodies in-database and
+      # builds the HVI over them (~7-9 min of init on top of DB create), so the
+      # container must not be declared unhealthy before init can finish.
+      retries: 80
       start_period: 120s
     deploy:
       resources:

labs/converged-database-lab/docker/Dockerfile.oracle

@@ -9,7 +9,10 @@ RUN chown -R oracle:oinstall /etc/ords
 
 # mongosh: used to verify and exercise the MongoDB API (port 27017) from inside
 # the container — the validator harness runs *.js scripts through it.
-RUN printf '[mongodb-org-8.0]\nname=MongoDB Repository\nbaseurl=https://repo.mongodb.org/yum/redhat/8/mongodb-org/8.0/x86_64/\ngpgcheck=1\nenabled=1\ngpgkey=https://pgp.mongodb.com/server-8.0.asc\n' > /etc/yum.repos.d/mongodb-org-8.0.repo \
+# Arch-aware repo URL: CI builds on x86_64; local builds on Apple-silicon (aarch64)
+# podman machines get MongoDB's aarch64 el8 packages instead of failing dependency
+# resolution against the x86_64 repo.
+RUN printf '[mongodb-org-8.0]\nname=MongoDB Repository\nbaseurl=https://repo.mongodb.org/yum/redhat/8/mongodb-org/8.0/%s/\ngpgcheck=1\nenabled=1\ngpgkey=https://pgp.mongodb.com/server-8.0.asc\n' "$(uname -m)" > /etc/yum.repos.d/mongodb-org-8.0.repo \
  && microdnf install -y mongodb-mongosh \
  && microdnf clean all
 

labs/converged-database-lab/docker/init/06-seed.sql

@@ -4,6 +4,13 @@ ALTER SESSION SET CURRENT_SCHEMA = lab_user;
 DECLARE
   -- All item declarations must precede subprogram bodies (PLS-00103 otherwise).
   seed    NUMBER := 42;
+  -- Ticket volume. 10,000 tickets is deliberately past the point where the
+  -- cost-based optimizer chooses the IVF vector index over exact full-scan
+  -- distance for module 03's filtered FETCH APPROX query (the crossover
+  -- arrives by roughly 3,000 rows on the Free container with fresh stats) —
+  -- so the captured plan shows the index + in-plan predicate evaluation, not
+  -- a toy-size full scan.
+  c_tickets CONSTANT PLS_INTEGER := 10000;
   v_vec   VARCHAR2(400);
   v_norm  NUMBER;
   TYPE t_f IS TABLE OF NUMBER INDEX BY PLS_INTEGER;
@@ -18,13 +25,71 @@ DECLARE
   v_prod  PLS_INTEGER;
   v_qty   PLS_INTEGER;
   v_ts    TIMESTAMP;
+  v_cat   PLS_INTEGER;
+  v_body  VARCHAR2(2000);
+  -- Enriched ticket-body corpus for tickets 301..c_tickets: 6 intros x 8 issue
+  -- categories x 4 details x 50 products x 5 closers ~= 48k distinct bodies, so
+  -- the 384-dim embeddings form real semantic clusters instead of 4 duplicated
+  -- points (matters for a meaningful IVF index and hybrid-search demo).
+  TYPE t_v IS TABLE OF VARCHAR2(400);
+  intro   t_v := t_v('Customer reports an issue with their recent experience.',
+                     'Contacting support after several attempts to resolve this alone.',
+                     'Opening a ticket on behalf of the account holder.',
+                     'Follow-up on an earlier conversation with the support team.',
+                     'New request submitted through the help center.',
+                     'Escalated from chat: the problem is still not resolved.');
+  refund  t_v := t_v('The package arrived damaged and a refund is requested.',
+                     'The box was crushed in transit and the item inside is damaged beyond use.',
+                     'Item arrived with a cracked casing; customer asks for their money back.',
+                     'Shipment was left in the rain and the contents are damaged and unusable.');
+  login   t_v := t_v('Login fails with an authentication error after reset.',
+                     'Password reset completed but sign-in still rejects the new credentials.',
+                     'Two-factor prompt never arrives and access to the account is blocked.',
+                     'Account is locked out after the security update and recovery does not work.');
+  shipdl  t_v := t_v('The order has not arrived within the promised window.',
+                     'Tracking has shown no movement for a week and the delivery date passed.',
+                     'Carrier marked the parcel delivered but nothing has arrived.',
+                     'Estimated delivery keeps slipping and no update has been provided.');
+  warr    t_v := t_v('Customer asks whether the warranty covers accidental damage.',
+                     'Requesting warranty service for a unit that stopped powering on.',
+                     'Screen developed dead pixels within the coverage period.',
+                     'Asking how to start a warranty claim for a failed component.');
+  billing t_v := t_v('The statement shows a duplicate charge for a single order.',
+                     'A promotional discount was not applied at checkout.',
+                     'The invoice total does not match the order confirmation email.',
+                     'Customer was billed after cancelling the subscription.');
+  exch    t_v := t_v('Requesting an exchange for a different size of the same product.',
+                     'The wrong color variant was delivered and a swap is requested.',
+                     'Received a different model than the one ordered.',
+                     'Asking to exchange a gift for store credit instead.');
+  install t_v := t_v('The setup guide does not match the ports on the device.',
+                     'Firmware update fails at forty percent every attempt.',
+                     'The companion app cannot discover the device during pairing.',
+                     'Installation completes but the device reboots continuously.');
+  perf    t_v := t_v('The device becomes very slow after a few hours of use.',
+                     'Battery drains overnight even when the unit is idle.',
+                     'Audio cuts out intermittently during playback.',
+                     'The connection drops whenever a second device joins.');
+  closing t_v := t_v('Customer asks for a response today.',
+                     'This is the second time the issue has been reported.',
+                     'The customer mentions they are a long-time subscriber.',
+                     'A callback number has been left on the account.',
+                     'No workaround has been found so far.');
   FUNCTION nxt RETURN NUMBER IS  -- LCG: deterministic across runs
   BEGIN
     seed := MOD(seed * 1103515245 + 12345, 2147483648);
     RETURN seed / 2147483648;
   END;
   FUNCTION pick(n IN PLS_INTEGER) RETURN PLS_INTEGER IS
   BEGIN RETURN TRUNC(nxt * n) + 1; END;
+  FUNCTION cat_detail(p_cat PLS_INTEGER) RETURN VARCHAR2 IS
+  BEGIN
+    RETURN CASE p_cat
+      WHEN 1 THEN refund (pick(4)) WHEN 2 THEN login(pick(4))
+      WHEN 3 THEN shipdl (pick(4)) WHEN 4 THEN warr (pick(4))
+      WHEN 5 THEN billing(pick(4)) WHEN 6 THEN exch (pick(4))
+      WHEN 7 THEN install(pick(4)) ELSE perf(pick(4)) END;
+  END;
 BEGIN
   FOR i IN 1..200 LOOP
     INSERT INTO customers (email, full_name, segment)
@@ -95,6 +160,9 @@ BEGIN
     UPDATE orders SET total_amount = ROUND(v_total,2) WHERE order_id = i;
   END LOOP;
 
+  -- Tickets 1..300: the ORIGINAL 4-template corpus, byte-for-byte (module 01's
+  -- ticket #1 proofs and module 03's read-after-write probe depend on ticket #1
+  -- keeping its "Login fails with an authentication error after reset." body).
   FOR i IN 1..300 LOOP
     v_norm := 0;
     FOR d IN 1..8 LOOP f(d) := nxt*2 - 1; v_norm := v_norm + f(d)*f(d); END LOOP;
@@ -119,6 +187,38 @@ BEGIN
             CASE WHEN MOD(i,3)=0 THEN 'closed' WHEN MOD(i,7)=0 THEN 'pending' ELSE 'open' END,
             TO_VECTOR(v_vec, 8, FLOAT32));
   END LOOP;
+
+  -- Tickets 301..c_tickets: the enriched scaled corpus. The LCG is re-seeded to
+  -- a fixed value here so this block is deterministic regardless of how the
+  -- upstream seeding evolves — same bodies on every fresh build.
+  seed := 342;
+  FOR i IN 301..c_tickets LOOP
+    v_cat  := pick(8);
+    v_cust := pick(200);
+    v_body := intro(pick(6))||' '||cat_detail(v_cat)||
+              ' Relates to Product '||pick(50)||'. '||closing(pick(5));
+    v_norm := 0;
+    FOR d IN 1..8 LOOP f(d) := nxt*2 - 1; v_norm := v_norm + f(d)*f(d); END LOOP;
+    v_norm := SQRT(v_norm);
+    v_vec := '[';
+    FOR d IN 1..8 LOOP
+      v_vec := v_vec || TO_CHAR(ROUND(f(d)/v_norm, 6),'FM990.999999') || CASE WHEN d<8 THEN ',' END;
+    END LOOP;
+    v_vec := v_vec || ']';
+    INSERT INTO support_tickets (customer_id, subject, body, status, embedding)
+    VALUES (v_cust,
+            CASE v_cat WHEN 1 THEN 'Refund request for damaged item'
+                       WHEN 2 THEN 'Cannot sign in after password reset'
+                       WHEN 3 THEN 'Shipping delay on recent order'
+                       WHEN 4 THEN 'Warranty question for product'
+                       WHEN 5 THEN 'Billing discrepancy on statement'
+                       WHEN 6 THEN 'Exchange request for recent order'
+                       WHEN 7 THEN 'Device setup and installation issue'
+                       ELSE 'Performance problem with product' END || ' #'||i,
+            v_body,
+            CASE WHEN MOD(i,3)=0 THEN 'closed' WHEN MOD(i,7)=0 THEN 'pending' ELSE 'open' END,
+            TO_VECTOR(v_vec, 8, FLOAT32));
+  END LOOP;
   COMMIT;
 END;
 /

labs/converged-database-lab/docker/init/08-vector-model.sql

@@ -0,0 +1,97 @@
+ALTER SESSION SET CONTAINER = FREEPDB1;
+
+-- In-database embeddings infrastructure (foundational; used by the AI articles).
+-- Runs once on first boot (gvenzl initdb), after 01-07. Loads Oracle's prebuilt
+-- augmented all-MiniLM-L12-v2 ONNX model (baked into the image at /opt/oracle/models
+-- by docker/Dockerfile.oracle), embeds all 10,000 ticket bodies into a real
+-- VECTOR(384) column, builds an IVF vector index on it, and gathers optimizer
+-- stats so the CBO's index-vs-scan choice is deterministic. Idempotent: every
+-- CREATE is guarded so a recreated container over a persistent volume re-runs
+-- cleanly.
+--
+-- Grants used by the AI modules beyond the base 01-lab-user.sql set:
+--   CREATE MINING MODEL  — DBMS_VECTOR.LOAD_ONNX_MODEL registers an ONNX model
+--   READ ON ONNX_MODELS  — read the baked .onnx from the image directory
+--   EXECUTE DBMS_VECTOR  — VECTOR_EMBEDDING / model load
+--   EXECUTE DBMS_RLS     — VPD policies (permission-aware retrieval proofs)
+--   CREATE/DROP ANY CONTEXT — application context for VPD tenant/user predicates
+GRANT CREATE MINING MODEL TO lab_user;
+GRANT CREATE ANY CONTEXT TO lab_user;
+GRANT DROP ANY CONTEXT TO lab_user;
+GRANT EXECUTE ON DBMS_VECTOR TO lab_user;
+GRANT EXECUTE ON DBMS_RLS TO lab_user;
+
+-- Directory over the baked model path; READ for lab_user so the load can open it.
+CREATE OR REPLACE DIRECTORY ONNX_MODELS AS '/opt/oracle/models';
+GRANT READ ON DIRECTORY ONNX_MODELS TO lab_user;
+
+ALTER SESSION SET CURRENT_SCHEMA = lab_user;
+
+-- Load the model into the lab_user schema as MINILM_L12 so VECTOR_EMBEDDING
+-- (MINILM_L12 USING <text> AS data) resolves for lab_user. The augmented model
+-- carries its own tokenizer + pooling, so the metadata only needs to declare the
+-- embedding function and the input attribute name (DATA, matching the AS data
+-- alias used everywhere downstream). DROP first for idempotent re-runs.
+DECLARE
+  v_model_path VARCHAR2(400);
+BEGIN
+  BEGIN
+    DBMS_VECTOR.DROP_ONNX_MODEL(model_name => 'MINILM_L12', force => TRUE);
+  EXCEPTION WHEN OTHERS THEN NULL; /* not yet loaded — nothing to drop */
+  END;
+  DBMS_VECTOR.LOAD_ONNX_MODEL(
+    directory  => 'ONNX_MODELS',
+    file_name  => 'all_MiniLM_L12_v2.onnx',
+    model_name => 'MINILM_L12',
+    metadata   => JSON('{"function":"embedding","embeddingOutput":"embedding","input":{"input":["DATA"]}}'));
+END;
+/
+
+-- Real 384-dim embedding column. The module-01 VECTOR(8) `embedding` column and
+-- its IVF index (ticket_vec_idx) are LEFT UNTOUCHED. Guarded so a recreated
+-- container does not error on the already-present column.
+DECLARE
+  v_exists NUMBER;
+BEGIN
+  SELECT COUNT(*) INTO v_exists FROM user_tab_columns
+   WHERE table_name = 'SUPPORT_TICKETS' AND column_name = 'BODY_VEC';
+  IF v_exists = 0 THEN
+    EXECUTE IMMEDIATE 'ALTER TABLE support_tickets ADD (body_vec VECTOR(384, FLOAT32))';
+  END IF;
+END;
+/
+
+-- Embed all 10,000 ticket bodies in-database — no external embedding service, no
+-- copy pipeline. ~3 min on the Free container (~55 embeddings/s, single-threaded).
+-- Re-runnable: recomputes from body.
+UPDATE support_tickets SET body_vec = VECTOR_EMBEDDING(MINILM_L12 USING body AS data);
+
+COMMIT;
+
+-- IVF (NEIGHBOR PARTITIONS) vector index on the real 384-dim column. IVF needs
+-- NO Vector Pool / VECTOR_MEMORY_SIZE, so it builds within the Free container's
+-- SGA. Guarded for idempotent re-runs.
+DECLARE
+BEGIN
+  EXECUTE IMMEDIATE 'DROP INDEX tickets_bodyvec_ivf';
+EXCEPTION WHEN OTHERS THEN NULL; /* absent — first run */
+END;
+/
+CREATE VECTOR INDEX tickets_bodyvec_ivf ON support_tickets(body_vec)
+  ORGANIZATION NEIGHBOR PARTITIONS
+  DISTANCE COSINE;
+
+-- Fresh optimizer statistics on the tables module 03's filtered-ANN plan spans.
+-- Load-bearing for determinism: without stats the CBO's dynamic sampling can pick
+-- the IVF plan even at toy sizes; with stats the choice is honestly cost-based —
+-- full-scan exact distance at hundreds of rows, the IVF index + in-plan filter
+-- predicates at 10,000 (the crossover arrives by roughly 3,000 rows on the
+-- Free container).
+-- (schema named explicitly: this script runs as SYS with CURRENT_SCHEMA=lab_user,
+-- and DBMS_STATS resolves USER to the logon user, not the current schema)
+BEGIN
+  DBMS_STATS.GATHER_TABLE_STATS('LAB_USER', 'SUPPORT_TICKETS');
+  DBMS_STATS.GATHER_TABLE_STATS('LAB_USER', 'CUSTOMERS');
+  DBMS_STATS.GATHER_TABLE_STATS('LAB_USER', 'ORDERS');
+END;
+/

labs/converged-database-lab/docker/init/09-hybrid-index.sql

@@ -0,0 +1,36 @@
+ALTER SESSION SET CONTAINER = FREEPDB1;
+ALTER SESSION SET CURRENT_SCHEMA = lab_user;
+
+-- Hybrid Vector Index on the ticket body: one domain index unifying Oracle Text
+-- (keyword) and a vector index (semantic) over the same column, queried via
+-- DBMS_HYBRID_VECTOR.SEARCH with RRF/score fusion (module 03, 04-hybrid-search.sql).
+-- Leaf-3-specific: the foundational model + body_vec + IVF live in 08 (on main);
+-- this HVI is the article-3 hybrid-search showcase and stays on this branch.
+--
+-- COLLISION NOTE: a CONTEXT/SEARCH text index and the HVI's internal text
+-- component are the SAME indextype, so two cannot sit on the same column
+-- (ORA-29879). The shared 05-text-vector.sql creates ticket_text_idx (a SEARCH
+-- index) on body; this module needs the richer body text for a meaningful hybrid
+-- proof, so we DROP ticket_text_idx here and let the HVI own body's text search.
+-- The HVI exposes the same CONTAINS(body, ...) capability, so keyword-only search
+-- still works. (On the eventual merge with the module-02 branch, whose
+-- read-after-write SEARCH proof uses ticket_text_idx, that proof moves to the HVI
+-- or to the subject column — a merge-time decision.)
+--
+-- VECTOR_IDXTYPE IVF keeps the HVI's vector component off the Vector Pool
+-- (Free-safe); MEMORY 256M caps the Oracle Text build memory modestly. Builds in
+-- ~6s on the Free container.
+DECLARE
+BEGIN
+  EXECUTE IMMEDIATE 'DROP INDEX tickets_hvi';
+EXCEPTION WHEN OTHERS THEN NULL; /* absent — first run */
+END;
+/
+DECLARE
+BEGIN
+  EXECUTE IMMEDIATE 'DROP INDEX ticket_text_idx';
+EXCEPTION WHEN OTHERS THEN NULL; /* HVI takes over body text search */
+END;
+/
+CREATE HYBRID VECTOR INDEX tickets_hvi ON support_tickets(body)
+  PARAMETERS('MODEL MINILM_L12 VECTOR_IDXTYPE IVF MEMORY 256M');