Skip to content

Commit 825d11a

Browse files
paulirwinclaude
paulirwin
and
claude
authored
Fix $filter on searchable string fields (#27) (#29)
Searchable Edm.String fields were indexed only as analyzed TextFields, so an exact TermQuery built from the filter literal never matched the lowercased, tokenized terms. Filterable/sortable/facetable searchable string fields now also emit a non-analyzed StringField under the same name, matching Azure's semantics where filters run on raw, non-analyzed content (https://learn.microsoft.com/azure/search/search-analyzers). Also: reject filters against fields with Filterable=false, and fix MergeDocument so dual-indexed fields aren't accidentally dropped during upsert. Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
1 parent 8f23e0b commit 825d11a

5 files changed

Lines changed: 262 additions & 14 deletions

File tree

‎AzureSearchEmulator.IntegrationTests/EmulatorIntegrationTests.cs‎

Lines changed: 85 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,90 @@ public async Task SearchDocuments_WithFilter_ShouldReturnResults()
149149
await indexClient.DeleteIndexAsync(indexName);
150150
}
151151

152+
[Fact]
153+
public async Task SearchDocuments_WithFilter_OnSearchableStringField_ShouldReturnResults()
154+
{
155+
// Regression test for issue #27: filtering a Searchable+Filterable string field
156+
// must work. Before the fix, the analyzer lowercased 'Electronics' at index time
157+
// and the exact TermQuery at filter time found zero hits.
158+
const string indexName = "test-filter-searchable-string";
159+
var indexClient = factory.CreateSearchIndexClient();
160+
var searchClient = factory.CreateSearchClient(indexName);
161+
162+
await CreateIndexAsync(indexClient, indexName);
163+
await UploadDocumentsAsync(searchClient);
164+
165+
var options = new SearchOptions
166+
{
167+
Filter = "Category eq 'Electronics'",
168+
Size = 50
169+
};
170+
var results = await searchClient.SearchAsync<Product>("*", options);
171+
172+
Assert.NotNull(results);
173+
var items = await results.Value.GetResultsAsync().ToListAsync();
174+
Assert.NotEmpty(items);
175+
Assert.True(items.All(r => r.Document.Category == "Electronics"),
176+
"All results should have Category = Electronics");
177+
178+
await indexClient.DeleteIndexAsync(indexName);
179+
}
180+
181+
[Fact]
182+
public async Task SearchDocuments_SearchOnDualIndexedField_StillMatchesAnalyzedTokens()
183+
{
184+
// Dual-indexing (TextField + StringField under same name) must not break
185+
// full-text search: a tokenized search for 'electronics' must still match.
186+
const string indexName = "test-search-on-dual-indexed";
187+
var indexClient = factory.CreateSearchIndexClient();
188+
var searchClient = factory.CreateSearchClient(indexName);
189+
190+
await CreateIndexAsync(indexClient, indexName);
191+
await UploadDocumentsAsync(searchClient);
192+
193+
var options = new SearchOptions { SearchFields = { "Category" }, Size = 50 };
194+
var results = await searchClient.SearchAsync<Product>("electronics", options);
195+
var items = await results.Value.GetResultsAsync().ToListAsync();
196+
197+
Assert.NotEmpty(items);
198+
Assert.True(items.All(r => r.Document.Category == "Electronics"));
199+
200+
await indexClient.DeleteIndexAsync(indexName);
201+
}
202+
203+
[Fact]
204+
public async Task SearchDocuments_FilterOnNonFilterableField_ShouldError()
205+
{
206+
// Azure rejects $filter against a non-filterable field; the emulator should too.
207+
const string indexName = "test-filter-nonfilterable";
208+
var indexClient = factory.CreateSearchIndexClient();
209+
var searchClient = factory.CreateSearchClient(indexName);
210+
211+
var index = new SearchIndex(indexName)
212+
{
213+
Fields =
214+
[
215+
new SearchField(nameof(Product.Id), SearchFieldDataType.String) { IsKey = true, IsStored = true, IsSearchable = true, IsFilterable = true },
216+
new SearchField(nameof(Product.Name), SearchFieldDataType.String) { IsSearchable = true, IsFilterable = false, IsStored = true },
217+
new SearchField(nameof(Product.Description), SearchFieldDataType.String) { IsSearchable = true, IsStored = true },
218+
new SearchField(nameof(Product.Price), SearchFieldDataType.Double) { IsFilterable = true, IsSortable = true, IsStored = true },
219+
new SearchField(nameof(Product.Category), SearchFieldDataType.String) { IsSearchable = true, IsFilterable = true, IsStored = true },
220+
new SearchField(nameof(Product.InStock), SearchFieldDataType.Boolean) { IsFilterable = true, IsStored = true }
221+
]
222+
};
223+
await indexClient.CreateIndexAsync(index);
224+
await UploadDocumentsAsync(searchClient);
225+
226+
var options = new SearchOptions { Filter = "Name eq 'Laptop Pro 15'", Size = 50 };
227+
await Assert.ThrowsAnyAsync<Exception>(async () =>
228+
{
229+
var results = await searchClient.SearchAsync<Product>("*", options);
230+
await results.Value.GetResultsAsync().ToListAsync();
231+
});
232+
233+
await indexClient.DeleteIndexAsync(indexName);
234+
}
235+
152236
[Fact]
153237
public async Task SearchDocuments_WithSorting_ShouldReturnSortedResults()
154238
{
@@ -464,7 +548,7 @@ private static async Task<SearchIndex> CreateIndexAsync(SearchIndexClient indexC
464548
new SearchField(nameof(Product.Name), SearchFieldDataType.String) { IsSearchable = true, IsStored = true },
465549
new SearchField(nameof(Product.Description), SearchFieldDataType.String) { IsSearchable = true, IsStored = true},
466550
new SearchField(nameof(Product.Price), SearchFieldDataType.Double) { IsFilterable = true, IsSortable = true, IsStored = true },
467-
new SearchField(nameof(Product.Category), SearchFieldDataType.String) { IsFilterable = true, IsStored = true },
551+
new SearchField(nameof(Product.Category), SearchFieldDataType.String) { IsSearchable = true, IsFilterable = true, IsStored = true },
468552
new SearchField(nameof(Product.InStock), SearchFieldDataType.Boolean) { IsFilterable = true, IsStored = true }
469553
]
470554
};

‎AzureSearchEmulator.UnitTests/LuceneNetIndexSearcherTests.cs‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
using System.Text.Json.Nodes;
2+
using AzureSearchEmulator.Indexing;
13
using AzureSearchEmulator.Models;
24
using AzureSearchEmulator.SearchData;
35
using AzureSearchEmulator.Searching;
@@ -928,3 +930,132 @@ private class StubIndexReaderFactory(Lucene.Net.Store.Directory directory) : ILu
928930
public void ClearCachedReader(string indexName) { }
929931
}
930932
}
933+
934+
/// <summary>
935+
/// Tests covering filter behavior for string fields indexed through the real
936+
/// SearchFieldExtensions.CreateField path (which applies the per-field analyzer),
937+
/// exercising combinations of Searchable and Filterable. See issue #27.
938+
/// </summary>
939+
public class LuceneNetIndexSearcher_SearchableFilterableTests
940+
{
941+
private static (LuceneTestHelper helper, LuceneNetIndexSearcher searcher) Build(SearchIndex index, List<JsonObject> docs)
942+
{
943+
var luceneDocs = docs.Select(d =>
944+
{
945+
var doc = new Lucene.Net.Documents.Document();
946+
foreach (var field in index.Fields)
947+
{
948+
if (d[field.Name] is { } value)
949+
{
950+
foreach (var f in field.CreateFields(value))
951+
{
952+
doc.Add(f);
953+
}
954+
}
955+
}
956+
return doc;
957+
}).ToList();
958+
959+
var helper = new LuceneTestHelper(index, luceneDocs);
960+
var searcher = new LuceneNetIndexSearcher(new StubReaderFactory(helper.Directory));
961+
return (helper, searcher);
962+
}
963+
964+
[Fact]
965+
public async Task Filter_StringEquality_SearchableAndFilterable_ReturnsMatch()
966+
{
967+
var index = new SearchIndex
968+
{
969+
Name = "products",
970+
Fields =
971+
[
972+
new SearchField { Name = "Id", Type = "Edm.String", Key = true, Searchable = false, Filterable = true },
973+
new SearchField { Name = "Category", Type = "Edm.String", Searchable = true, Filterable = true },
974+
]
975+
};
976+
var docs = new List<JsonObject>
977+
{
978+
new() { ["Id"] = "1", ["Category"] = "Electronics" },
979+
new() { ["Id"] = "2", ["Category"] = "Books" },
980+
};
981+
var (helper, searcher) = Build(index, docs);
982+
using var _ = helper;
983+
984+
var response = await searcher.Search(index, new SearchRequest
985+
{
986+
Search = "*",
987+
Filter = "Category eq 'Electronics'",
988+
Top = 50
989+
});
990+
991+
Assert.Single(response.Results);
992+
Assert.Equal("1", response.Results[0]["Id"]?.GetValue<string>());
993+
}
994+
995+
[Fact]
996+
public async Task Filter_StringEquality_Defaults_ReturnsMatch()
997+
{
998+
// Mirrors the bug reported in issue #27: fields where Searchable and Filterable
999+
// are unset (both default to true for Edm.String) must still filter correctly.
1000+
var index = new SearchIndex
1001+
{
1002+
Name = "products",
1003+
Fields =
1004+
[
1005+
new SearchField { Name = "Id", Type = "Edm.String", Key = true, Searchable = true, Filterable = true },
1006+
new SearchField { Name = "Category", Type = "Edm.String" },
1007+
]
1008+
};
1009+
var docs = new List<JsonObject>
1010+
{
1011+
new() { ["Id"] = "1", ["Category"] = "Electronics" },
1012+
new() { ["Id"] = "2", ["Category"] = "Books" },
1013+
};
1014+
var (helper, searcher) = Build(index, docs);
1015+
using var _ = helper;
1016+
1017+
var response = await searcher.Search(index, new SearchRequest
1018+
{
1019+
Search = "*",
1020+
Filter = "Category eq 'Electronics'",
1021+
Top = 50
1022+
});
1023+
1024+
Assert.Single(response.Results);
1025+
Assert.Equal("1", response.Results[0]["Id"]?.GetValue<string>());
1026+
}
1027+
1028+
[Fact]
1029+
public async Task Filter_StringEquality_SearchableNotFilterable_ShouldError()
1030+
{
1031+
var index = new SearchIndex
1032+
{
1033+
Name = "posts",
1034+
Fields =
1035+
[
1036+
new SearchField { Name = "Id", Type = "Edm.String", Key = true, Searchable = false, Filterable = true },
1037+
new SearchField { Name = "Body", Type = "Edm.String", Searchable = true, Filterable = false },
1038+
]
1039+
};
1040+
var docs = new List<JsonObject>
1041+
{
1042+
new() { ["Id"] = "1", ["Body"] = "Hello world" },
1043+
};
1044+
var (helper, searcher) = Build(index, docs);
1045+
using var _ = helper;
1046+
1047+
await Assert.ThrowsAnyAsync<Exception>(() => searcher.Search(index, new SearchRequest
1048+
{
1049+
Search = "*",
1050+
Filter = "Body eq 'Hello world'",
1051+
Top = 50
1052+
}));
1053+
}
1054+
1055+
private class StubReaderFactory(Lucene.Net.Store.Directory directory) : ILuceneIndexReaderFactory
1056+
{
1057+
public IndexReader GetIndexReader(string indexName) => DirectoryReader.Open(directory);
1058+
public IndexReader RefreshReader(string indexName) => GetIndexReader(indexName);
1059+
public void ClearCachedReader(string indexName) { }
1060+
}
1061+
}

‎AzureSearchEmulator/Indexing/SearchFieldExtensions.cs‎

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,35 @@ namespace AzureSearchEmulator.Indexing;
77

88
public static class SearchFieldExtensions
99
{
10-
public static IIndexableField CreateField(this SearchField field, JsonNode value)
10+
public static IEnumerable<IIndexableField> CreateFields(this SearchField field, JsonNode value)
1111
{
1212
var stored = field.Retrievable ? Field.Store.YES : Field.Store.NO;
1313

14-
return field.Type switch
14+
if (field.Type == "Edm.String")
15+
{
16+
var str = value.GetValue<string>();
17+
var searchable = field.Searchable.GetValueOrDefault(true);
18+
var filterable = field.Filterable;
19+
20+
if (searchable)
21+
{
22+
yield return new TextField(field.Name, str, stored);
23+
// Filter/sort/facet require a non-analyzed copy under the same field name
24+
// so TermQuery-based filters match the raw literal (matches Azure semantics).
25+
if (filterable || field.Sortable.GetValueOrDefault() || field.Facetable.GetValueOrDefault())
26+
{
27+
yield return new StringField(field.Name, str, Field.Store.NO);
28+
}
29+
}
30+
else
31+
{
32+
yield return new StringField(field.Name, str, stored);
33+
}
34+
yield break;
35+
}
36+
37+
yield return field.Type switch
1538
{
16-
"Edm.String" when field.Searchable.GetValueOrDefault(true) => new TextField(field.Name, value.GetValue<string>(), stored),
17-
"Edm.String" when !field.Searchable.GetValueOrDefault(true) => new StringField(field.Name, value.GetValue<string>(), stored),
1839
"Edm.Int32" => new Int32Field(field.Name, value.GetValue<int>(), stored),
1940
"Edm.Int64" => new Int64Field(field.Name, value.GetValue<long>(), stored),
2041
"Edm.Double" => new DoubleField(field.Name, value.GetValue<double>(), stored),

‎AzureSearchEmulator/Indexing/UpsertIndexDocumentActionBase.cs‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,8 @@ protected IEnumerable<IIndexableField> GetDocFields(SearchIndex index)
3333
return from f in index.Fields
3434
join v in Item on f.Name equals v.Key
3535
where v.Value != null
36-
select f.CreateField(v.Value!); // [!]: null checked by where clause
36+
from indexField in f.CreateFields(v.Value!) // [!]: null checked by where clause
37+
select indexField;
3738
}
3839

3940
protected static void MergeDocument(IndexingContext context, Term keyTerm, IEnumerable<IIndexableField> docFields, bool uploadIfMissing)
@@ -50,15 +51,13 @@ protected static void MergeDocument(IndexingContext context, Term keyTerm, IEnum
5051

5152
var doc = docs.TotalHits == 0 ? new Document() : searcher.Doc(docs.ScoreDocs[0].Doc);
5253

53-
foreach (var docField in docFields)
54+
var materialized = docFields.ToList();
55+
foreach (var name in materialized.Select(f => f.Name).Distinct())
56+
{
57+
doc.RemoveFields(name);
58+
}
59+
foreach (var docField in materialized)
5460
{
55-
var field = doc.GetField(docField.Name);
56-
57-
if (field != null)
58-
{
59-
doc.RemoveField(docField.Name);
60-
}
61-
6261
doc.Add(docField);
6362
}
6463

‎AzureSearchEmulator/Searching/ODataQueryVisitor.cs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ public Query Visit(BinaryOperatorToken tokenIn)
4848
Right: LiteralToken literalToken
4949
})
5050
{
51+
EnsureFilterable(path);
5152
return tokenIn.OperatorKind switch
5253
{
5354
BinaryOperatorKind.Equal => HandleEqualComparison(path, literalToken),
@@ -67,6 +68,7 @@ public Query Visit(BinaryOperatorToken tokenIn)
6768
Right: LiteralToken negatedLiteral
6869
})
6970
{
71+
EnsureFilterable(negatedPath);
7072
var equalQuery = tokenIn.OperatorKind switch
7173
{
7274
BinaryOperatorKind.Equal => HandleEqualComparison(negatedPath, negatedLiteral),
@@ -91,6 +93,17 @@ public Query Visit(BinaryOperatorToken tokenIn)
9193
throw new NotImplementedException();
9294
}
9395

96+
private void EnsureFilterable(string path)
97+
{
98+
if (_index is null) return;
99+
var field = _index.Fields.FirstOrDefault(f => string.Equals(f.Name, path, StringComparison.OrdinalIgnoreCase));
100+
if (field is null) return;
101+
if (!field.Filterable)
102+
{
103+
throw new InvalidOperationException($"Field '{field.Name}' is not filterable.");
104+
}
105+
}
106+
94107
private static Occur GetOccurFromOperator(BinaryOperatorKind operatorKind)
95108
{
96109
return operatorKind switch

0 commit comments

Comments
 (0)