Refresh vector search samples for March 2026 product behavior

.gitignore

@@ -406,3 +406,9 @@ FodyWeavers.xsd
 /DotNet/SqlClient/Properties/launchSettings Empty.json
 /DotNet/SqlClient/Properties/launchSettingsProd.json
 /DotNet/SqlClient/Properties/launchSettings.prod.json
+
+# macOS
+.DS_Store
+
+# Local session prep (VSLive 2026 · Beyond Embeddings)
+VSLive2026/

DiskANN/FineFoodReviews/000-setup.sql

@@ -0,0 +1,64 @@
+/*
+    Fine Food Reviews · 000-setup
+    ------------------------------------------
+    Creates the reviews table + external model + external table for CSV load.
+
+    Pre-req: PREVIEW_FEATURES = ON at the database scope.
+             Full-text search enabled on the database.
+             An Azure OpenAI text-embedding-3-small deployment reachable from the DB
+             via database-scoped credential + external URL endpoint.
+*/
+-- Uncomment if using SQL Server 2025:
+-- use FineFoodReviews;
+-- go
+
+-- ---------- reviews table ----------
+drop table if exists dbo.reviews;
+go
+
+create table dbo.reviews (
+    Id          int             not null constraint PK_reviews primary key,
+    Time        bigint          null,
+    ProductId   nvarchar(50)    null,
+    UserId      nvarchar(50)    null,
+    Score       tinyint         null,           -- 1..5
+    Summary     nvarchar(500)   null,           -- short review title
+    [Text]      nvarchar(max)   null,           -- long review body
+    combined    as (isnull(Summary,'') + N': ' + isnull([Text],'')) persisted,
+    embedding   vector(1536)    null            -- populated by 001-load-and-embed.sql
+);
+go
+
+-- ---------- external model (reuse if already present) ----------
+if not exists (select 1 from sys.external_models where name = N'AIEmbeddings')
+begin
+    print N'AIEmbeddings external model not found. Create it via the pattern in ../Wikipedia/001-setup-objects.sql';
+    -- Example (adjust endpoint, deployment name, and credential to your Azure OpenAI resource):
+    /*
+    create external model AIEmbeddings
+    with (
+        location    = 'https://<your-aoai>.openai.azure.com/openai/deployments/text-embedding-3-small/embeddings?api-version=2024-08-01-preview',
+        api_format  = 'Azure OpenAI',
+        model_type  = embeddings,
+        model       = 'text-embedding-3-small',
+        credential  = [https://<your-aoai>.openai.azure.com]
+    );
+    */
+end;
+go
+
+-- ---------- (optional) external data source for CSV load ----------
+-- Only needed if loading from Azure Blob Storage. Skip if loading via SqlBulkCopy from a client.
+--
+-- create database scoped credential [sample_data]
+--     with identity = 'Managed Identity';
+-- go
+-- create external data source [sample_data]
+--     with (
+--         type     = blob_storage,
+--         location = 'https://<myaccount>.blob.core.windows.net/sample-data/',
+--         credential = [sample_data]
+--     );
+-- go
+
+print N'Setup complete. Next: 001-load-and-embed.sql';

DiskANN/FineFoodReviews/001-load-and-embed.sql

@@ -0,0 +1,74 @@
+/*
+    Fine Food Reviews · 001-load-and-embed
+    ---------------------------------------------------
+    Loads the first 500 rows of Datasets/Reviews.csv into dbo.reviews, then
+    generates a VECTOR(1536) embedding for each row via AI_GENERATE_EMBEDDINGS.
+
+    Load options:
+      (A) Client-side load (RECOMMENDED for the demo — no blob storage needed).
+          Use SqlBulkCopy / bcp / Azure Data Studio's Import wizard to push
+          Datasets/Reviews.csv (first 500 rows) into dbo.reviews. Then jump to
+          "STEP 2: EMBED" below.
+
+      (B) Server-side load from Azure Blob Storage. Uncomment the BULK INSERT
+          block below if you've staged the CSV in the external data source
+          declared in 000-setup.sql.
+*/
+-- Uncomment if using SQL Server 2025:
+-- use FineFoodReviews;
+-- go
+
+-- ==========================================================================
+-- STEP 1 · LOAD  (option B — server-side; keep commented if using option A)
+-- ==========================================================================
+/*
+truncate table dbo.reviews;
+
+bulk insert dbo.reviews (Id, Time, ProductId, UserId, Score, Summary, [Text])
+from 'reviews/Reviews.csv'
+with (
+    data_source     = 'sample_data',
+    format          = 'csv',
+    firstrow        = 2,
+    codepage        = '65001',
+    fieldterminator = ',',
+    rowterminator   = '0x0a',
+    fieldquote      = '"',
+    batchsize       = 500,
+    lastrow         = 501,           -- keep it small for a live demo
+    tablock
+);
+go
+*/
+
+-- ==========================================================================
+-- STEP 2 · EMBED
+-- ==========================================================================
+-- Confirm row count
+select count(*) as loaded_rows,
+       sum(case when [Text] is not null then 1 else 0 end) as with_text
+from dbo.reviews;
+go
+
+-- Sanity check the external model
+select top (1)
+    ai_generate_embeddings(N'hello world' use model AIEmbeddings) as sample_embedding_bytes;
+go
+
+-- Bulk-embed all reviews. On 500 rows this is ~30 s over the network.
+update dbo.reviews
+set embedding = ai_generate_embeddings([combined] use model AIEmbeddings)
+where embedding is null
+  and [combined] is not null;
+go
+
+-- Verify: every row has an embedding
+select
+    count(*)                              as total_rows,
+    sum(iif(embedding is null, 1, 0))     as missing_embeddings,
+    min(datalength(embedding))            as min_bytes,
+    max(datalength(embedding))            as max_bytes    -- expected: 6152 bytes (1536 × 4 + 8-byte header)
+from dbo.reviews;
+go
+
+print N'Load + embed complete. Next: 002-diskann-and-fulltext.sql';

DiskANN/FineFoodReviews/002-diskann-and-fulltext.sql

@@ -0,0 +1,80 @@
+/*
+    Fine Food Reviews · 002-diskann-and-fulltext
+    ---------------------------------------------------------
+    Builds the two indexes hybrid search needs:
+      1. DiskANN vector index on dbo.reviews(embedding)          — for VECTOR_SEARCH
+      2. Full-text catalog + index on dbo.reviews(combined)      — for FREETEXTTABLE
+
+    Both indexes live on the same table. No data movement.
+*/
+-- Uncomment if using SQL Server 2025:
+-- use FineFoodReviews;
+-- go
+
+-- ==========================================================================
+-- 1 · DiskANN vector index
+-- ==========================================================================
+if exists (
+    select 1
+    from sys.indexes
+    where object_id = object_id(N'dbo.reviews') and name = N'vec_idx_reviews'
+)
+    drop index vec_idx_reviews on dbo.reviews;
+go
+
+create vector index vec_idx_reviews
+on dbo.reviews (embedding)
+with (
+    metric = 'cosine',
+    type   = 'diskann'
+);
+go
+
+-- Verify
+select
+    i.name         as index_name,
+    v.build_parameters,
+    json_value(v.build_parameters, '$.Version') as index_version
+from sys.vector_indexes v
+join sys.indexes i on v.object_id = i.object_id and v.index_id = i.index_id
+where v.object_id = object_id(N'dbo.reviews');
+go
+
+-- ==========================================================================
+-- 2 · Full-text catalog + index on combined (Summary + ': ' + Text)
+-- ==========================================================================
+if exists (select 1 from sys.fulltext_catalogs where name = N'ft_reviews_catalog')
+begin
+    if exists (
+        select 1 from sys.fulltext_indexes
+        where object_id = object_id(N'dbo.reviews')
+    )
+        drop fulltext index on dbo.reviews;
+
+    drop fulltext catalog ft_reviews_catalog;
+end;
+go
+
+create fulltext catalog ft_reviews_catalog as default;
+go
+
+create fulltext index on dbo.reviews (combined language 1033)
+    key index PK_reviews  -- named explicitly in 000-setup.sql
+    on ft_reviews_catalog
+    with change_tracking auto;
+go
+
+-- Wait for population (500 rows populates in ~10s). Verify:
+-- 0 = idle (populated); non-zero = still crawling.
+select
+    fulltextcatalogproperty(N'ft_reviews_catalog', 'PopulateStatus') as populate_status,
+    fulltextcatalogproperty(N'ft_reviews_catalog', 'ItemCount')      as item_count;
+go
+
+-- Smoke test full-text: how many reviews mention "oatmeal"?
+select count(*) as oatmeal_mentions
+from dbo.reviews
+where contains(combined, N'oatmeal');
+go
+
+print N'Indexes ready. Next: 003-hybrid-search.sql';

DiskANN/FineFoodReviews/003-hybrid-search.sql

@@ -0,0 +1,134 @@
+/*
+    Fine Food Reviews · 003-hybrid-search
+    ---------------------------------------------------
+    Three queries, back to back:
+        Q1  vector-only    — great at paraphrase, weak at SKUs / brand names
+        Q2  full-text only — great at exact matches, weak at paraphrase
+        Q3  HYBRID (RRF)   — both. Rows that win in either modality surface.
+
+    Same table, same query, three retrieval strategies. This is the whole point.
+*/
+-- Uncomment if using SQL Server 2025:
+-- use FineFoodReviews;
+-- go
+
+set statistics time on;
+set statistics io on;
+go
+
+-- --------------------------------------------------------------------------
+-- Q1 · VECTOR-ONLY  (paraphrase query)
+-- --------------------------------------------------------------------------
+-- Shopper intent: "smooth flavorful coffee"
+-- Reviewers rarely say "smooth" — they say "not bitter", "not burnt", "mellow",
+-- "easy on the stomach". Vectors bridge that. BM25 alone would miss most of it.
+declare @q1 nvarchar(500) = N'smooth flavorful coffee';
+declare @v1 vector(1536)  = ai_generate_embeddings(@q1 use model AIEmbeddings);
+
+select top (10) with approximate
+    r.Id,
+    r.ProductId,
+    r.Score,
+    r.Summary,
+    left(r.[Text], 120) + iif(len(r.[Text]) > 120, N'…', N'') as text_preview,
+    s.distance as cosine_distance
+from vector_search(
+    table       = dbo.reviews as r,
+    column      = embedding,
+    similar_to  = @v1,
+    metric      = 'cosine'
+) as s
+order by s.distance;
+go
+
+
+-- --------------------------------------------------------------------------
+-- Q2 · FULL-TEXT ONLY  (brand / SKU query)
+-- --------------------------------------------------------------------------
+-- Shopper intent: "Green Mountain Nantucket Blend" — an exact brand + product name.
+-- Vector alone drifts to generic Keurig / coffee reviews.
+-- BM25 nails the literal brand + blend match.
+select top (10)
+    r.Id,
+    r.ProductId,
+    r.Score,
+    r.Summary,
+    left(r.[Text], 120) + iif(len(r.[Text]) > 120, N'…', N'') as text_preview,
+    ftt.[rank] as bm25_rank
+from dbo.reviews r
+inner join freetexttable(dbo.reviews, combined, N'Green Mountain Nantucket Blend') ftt
+    on r.Id = ftt.[KEY]
+order by ftt.[rank] desc;
+go
+
+-- --------------------------------------------------------------------------
+-- Q3 · HYBRID  (vector + full-text, blended with RRF)
+-- --------------------------------------------------------------------------
+-- Shopper intent: "Green Mountain blend that is smooth and flavorful" — brand AND paraphrase.
+-- Keyword side locks onto "Green Mountain"; vector side finds reviewers writing
+-- "not bitter", "mellow", "easy on the stomach". RRF surfaces rows that win in
+-- either signal.
+declare @q  nvarchar(1000) = N'Green Mountain blend that is smooth and flavorful';
+declare @v  vector(1536)   = ai_generate_embeddings(@q use model AIEmbeddings);
+declare @k  int            = 20;   -- candidates per signal
+declare @rrf_k int          = 60;  -- standard RRF constant
+
+with keyword_search as (
+    select top (@k)
+        r.Id,
+        rank() over (order by ftt.[rank] desc) as keyword_rank
+    from dbo.reviews r
+    inner join freetexttable(dbo.reviews, combined, @q) ftt
+        on r.Id = ftt.[KEY]
+    order by ftt.[rank] desc
+),
+semantic_search as (
+    select top (@k)
+        s.Id,
+        rank() over (order by s.cosine_distance) as vector_rank
+    from (
+        select top (@k) with approximate
+            r.Id,
+            s0.distance as cosine_distance
+        from vector_search(
+            table       = dbo.reviews as r,
+            column      = embedding,
+            similar_to  = @v,
+            metric      = 'cosine'
+        ) as s0
+        order by s0.distance
+    ) s
+),
+fused as (
+    select
+        coalesce(ss.Id, ks.Id) as Id,
+        ss.vector_rank,
+        ks.keyword_rank,
+        coalesce(1.0 / (@rrf_k + ss.vector_rank),  0.0) +
+        coalesce(1.0 / (@rrf_k + ks.keyword_rank), 0.0) as rrf_score
+    from semantic_search ss
+    full outer join keyword_search ks on ss.Id = ks.Id
+)
+select top (7)
+    r.Id,
+    r.ProductId,
+    r.Score,
+    r.Summary,
+    left(r.[Text], 100) + iif(len(r.[Text]) > 100, N'…', N'') as text_preview,
+    rank() over (order by f.rrf_score desc) as rrf_rank,
+    f.vector_rank,
+    f.keyword_rank,
+    cast(f.rrf_score * 1000 as int) as rrf_score_x1000
+from fused f
+inner join dbo.reviews r on f.Id = r.Id
+order by f.rrf_score desc;
+go
+
+/*
+    Takeaway
+    ---------
+    Hybrid isn't a library. It's a WITH clause.
+    Both indexes live on the same table. No second data store.
+    Rerank is one REST call away when relevance beats latency — see
+    ../../Semantic-Reranking/02-rerank.sql.
+*/

DiskANN/FineFoodReviews/README.md

@@ -0,0 +1,29 @@
+# Fine Food Reviews — Hybrid Search
+
+Hybrid vector + full-text search over Amazon Fine Food Reviews. Runs on **Azure SQL** (Hyperscale or General Purpose) and **SQL Server 2025**.
+
+**Scenario:** e-commerce shoppers type BOTH paraphrases (*"healthy oatmeal for a picky eater"*) and product SKUs / brand names (*"Merrick Turducken"*). Vector search misses SKUs; full-text misses paraphrases. Hybrid reciprocal-rank fusion (RRF) surfaces rows that win in either signal.
+
+## Files (run in order once, then re-run only `003` to compare queries)
+
+| # | File | Purpose |
+|---|---|---|
+| 000 | [`000-setup.sql`](000-setup.sql) | Create the `reviews` table, external model, and (optionally) the source-data external table. |
+| 001 | [`001-load-and-embed.sql`](001-load-and-embed.sql) | Load 500 sample reviews from [`../../Datasets/Reviews.csv`](../../Datasets/Reviews.csv), then embed via `AI_GENERATE_EMBEDDINGS`. |
+| 002 | [`002-diskann-and-fulltext.sql`](002-diskann-and-fulltext.sql) | Build the DiskANN vector index + the full-text catalog/index on `combined`. |
+| 003 | [`003-hybrid-search.sql`](003-hybrid-search.sql) | Vector-only, full-text-only, and hybrid-RRF queries side by side. |
+
+Data source: [`../../Datasets/Reviews.csv`](../../Datasets/Reviews.csv) (500k Amazon Fine Food Reviews; the sample uses the first 500 rows). Pre-computed embeddings alternative: [`../../Datasets/FineFoodEmbeddings.csv`](../../Datasets/FineFoodEmbeddings.csv).
+
+## Timing on a 2-vCore Hyperscale instance (500 rows)
+
+- Embedding 500 rows: ~30 s (network to Azure OpenAI).
+- DiskANN build: ~2 s.
+- Full-text catalog build: ~10 s.
+- Hybrid query: ~150 ms.
+
+## Pre-reqs
+
+- An `AIEmbeddings` external model pointing at a `text-embedding-3-small` deployment. See [`../Wikipedia/001-setup-objects.sql`](../Wikipedia/001-setup-objects.sql) for the setup pattern.
+- Full-text search enabled on the database.
+- Optional Python helpers ([`_load-reviews.py`](_load-reviews.py), [`_embed-reviews.py`](_embed-reviews.py)) for loading the CSV directly from Python instead of via `OPENROWSET`.