Skip to content

Commit 2c6c825

Browse files
mgodwanmgodwan
committed
Enable lucene in search tests
Signed-off-by: Mohit Godwani <mgodwan@amazon.com>
1 parent 92df0c6 commit 2c6c825

34 files changed

Lines changed: 170 additions & 106 deletions

‎sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneFieldFactoryRegistry.java‎

Lines changed: 4 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@
1010

1111
import org.apache.lucene.document.Field;
1212
import org.apache.lucene.document.FieldType;
13-
import org.apache.lucene.document.LongPoint;
1413
import org.apache.lucene.document.SortedNumericDocValuesField;
14+
import org.apache.lucene.index.DocValuesType;
1515
import org.apache.lucene.index.IndexOptions;
1616
import org.apache.lucene.util.BytesRef;
1717
import org.opensearch.common.annotation.ExperimentalApi;
@@ -26,8 +26,6 @@
2626
import java.util.Set;
2727
import java.util.concurrent.ConcurrentHashMap;
2828

29-
import static org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.POINT_RANGE;
30-
3129
/**
3230
* Registry of {@link LuceneFieldFactory} instances keyed by OpenSearch field type name.
3331
*
@@ -46,6 +44,8 @@ public final class LuceneFieldFactoryRegistry {
4644
ID_FIELD_TYPE.setTokenized(false);
4745
ID_FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
4846
ID_FIELD_TYPE.setOmitNorms(true);
47+
ID_FIELD_TYPE.setStored(false);
48+
ID_FIELD_TYPE.setDocValuesType(DocValuesType.NONE);
4949
ID_FIELD_TYPE.freeze();
5050
}
5151

@@ -62,16 +62,12 @@ public final class LuceneFieldFactoryRegistry {
6262
doc.add(new Field(ft.name(), value.toString(), lft));
6363
};
6464

65-
private static final FieldType ID_FIELD_TYPE = buildIdFieldType();
66-
6765
private static final LuceneFieldFactory ID_FIELD_FACTORY = (doc, ft, value, lft) -> {
6866
doc.add(new Field(ft.name(), new BytesRef((byte[]) value), ID_FIELD_TYPE));
6967
};
7068

7169
private static final LuceneFieldFactory SEQ_NO_FIELD_FACTORY = (doc, ft, value, lft) -> {
72-
if (ft.getCapabilityMap().get(LucenePlugin.DATA_FORMAT).contains(POINT_RANGE)) {
73-
doc.add(new LongPoint(ft.name(), (long) value));
74-
}
70+
// do nothing for now since we don't want to index seq no indexing without soft deletes enabled.
7571
};
7672

7773
// ── Registry ──
@@ -124,11 +120,4 @@ public LuceneFieldFactory get(String typeName) {
124120
public Set<String> supportedTypes() {
125121
return Set.copyOf(factories.keySet());
126122
}
127-
128-
private static FieldType buildIdFieldType() {
129-
FieldType ft = new FieldType(IdFieldMapper.Defaults.FIELD_TYPE);
130-
ft.setStored(false);
131-
ft.freeze();
132-
return ft;
133-
}
134123
}

‎sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneDocumentInput.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -119,8 +119,6 @@ private static FieldType getFieldType(MappedFieldType fieldType, Set<FieldTypeCa
119119
}
120120
luceneFieldType.setStored(false);
121121
luceneFieldType.setOmitNorms(true);
122-
} else {
123-
luceneFieldType = null;
124122
}
125123
return luceneFieldType;
126124
}

‎sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneDocumentInputTests.java‎

Lines changed: 11 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -9,31 +9,28 @@
99
package org.opensearch.be.lucene.index;
1010

1111
import org.apache.lucene.document.Document;
12-
import org.apache.lucene.document.FieldType;
1312
import org.apache.lucene.index.DocValuesType;
1413
import org.apache.lucene.index.IndexOptions;
1514
import org.apache.lucene.index.IndexableField;
1615
import org.apache.lucene.index.IndexableFieldType;
16+
import org.opensearch.be.lucene.LucenePlugin;
1717
import org.opensearch.index.mapper.IdFieldMapper;
18-
import org.opensearch.index.mapper.KeywordFieldMapper;
1918
import org.opensearch.index.mapper.MappedFieldType;
20-
import org.opensearch.index.mapper.MatchOnlyTextFieldMapper;
2119
import org.opensearch.index.mapper.SeqNoFieldMapper;
22-
import org.opensearch.index.mapper.TextFieldMapper;
23-
import org.opensearch.index.mapper.TextSearchInfo;
24-
import org.opensearch.test.OpenSearchTestCase;
2520

2621
import java.nio.charset.StandardCharsets;
27-
import java.util.Collections;
22+
import java.util.Map;
23+
import java.util.Set;
2824

25+
import static org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.FULL_TEXT_SEARCH;
2926
import static org.mockito.Mockito.mock;
3027
import static org.mockito.Mockito.when;
3128

3229
/**
3330
* Verifies that each field type registered in {@link org.opensearch.be.lucene.LuceneFieldFactoryRegistry}
3431
* produces Lucene fields with the expected storage properties.
3532
*/
36-
public class LuceneDocumentInputTests extends OpenSearchTestCase {
33+
public class LuceneDocumentInputTests extends LucenePluginBaseTests {
3734

3835
public void testIdFieldProperties() {
3936
MappedFieldType idField = mockIdField();
@@ -50,7 +47,7 @@ public void testIdFieldProperties() {
5047
}
5148

5249
public void testTextFieldProperties() {
53-
MappedFieldType textField = new TextFieldMapper.TextFieldType("content");
50+
MappedFieldType textField = mockTextField("content");
5451
LuceneDocumentInput input = new LuceneDocumentInput();
5552
input.addField(textField, "hello world");
5653

@@ -66,13 +63,7 @@ public void testTextFieldProperties() {
6663
}
6764

6865
public void testKeywordFieldProperties() {
69-
FieldType kwFieldType = new FieldType();
70-
kwFieldType.setTokenized(false);
71-
kwFieldType.setStored(false);
72-
kwFieldType.setOmitNorms(true);
73-
kwFieldType.setIndexOptions(IndexOptions.DOCS);
74-
kwFieldType.freeze();
75-
MappedFieldType keywordField = new KeywordFieldMapper.KeywordFieldType("status", kwFieldType);
66+
MappedFieldType keywordField = mockKeywordField("status");
7667

7768
LuceneDocumentInput input = new LuceneDocumentInput();
7869
input.addField(keywordField, "active");
@@ -89,13 +80,7 @@ public void testKeywordFieldProperties() {
8980
}
9081

9182
public void testMatchOnlyTextFieldProperties() {
92-
MappedFieldType matchOnlyField = new MatchOnlyTextFieldMapper.MatchOnlyTextFieldType(
93-
"body",
94-
true,
95-
false,
96-
new TextSearchInfo(TextFieldMapper.Defaults.FIELD_TYPE, null, null, null),
97-
Collections.emptyMap()
98-
);
83+
MappedFieldType matchOnlyField = mockMatchOnlyTextField("body");
9984

10085
LuceneDocumentInput input = new LuceneDocumentInput();
10186
input.addField(matchOnlyField, "some text");
@@ -108,7 +93,7 @@ public void testMatchOnlyTextFieldProperties() {
10893
assertFalse("match_only_text: should not be stored", ft.stored());
10994
assertTrue("match_only_text: should omit norms", ft.omitNorms());
11095
assertEquals("match_only_text: should have no doc values", DocValuesType.NONE, ft.docValuesType());
111-
assertNotEquals("match_only_text: should be indexed", IndexOptions.NONE, ft.indexOptions());
96+
assertNotEquals("match_only_text: should be indexed", IndexOptions.DOCS, ft.indexOptions());
11297
}
11398

11499
public void testSeqNoFieldProperties() {
@@ -118,19 +103,14 @@ public void testSeqNoFieldProperties() {
118103

119104
Document doc = input.getFinalInput();
120105
IndexableField field = doc.getField(SeqNoFieldMapper.NAME);
121-
assertNotNull("_seq_no field should be present in document", field);
122-
123-
// LongPoint: dimensional field, not stored, not indexed via inverted index
124-
IndexableFieldType ft = field.fieldType();
125-
assertFalse("_seq_no: should not be stored", ft.stored());
126-
assertEquals("_seq_no: LongPoint has no inverted index", IndexOptions.NONE, ft.indexOptions());
127-
assertTrue("_seq_no: should have point dimensions", ft.pointDimensionCount() > 0);
106+
assertNull("_seq_no field should be present in document", field);
128107
}
129108

130109
private static MappedFieldType mockIdField() {
131110
MappedFieldType idField = mock(MappedFieldType.class);
132111
when(idField.typeName()).thenReturn(IdFieldMapper.CONTENT_TYPE);
133112
when(idField.name()).thenReturn(IdFieldMapper.NAME);
113+
when(idField.getCapabilityMap()).thenReturn(Map.of(LucenePlugin.DATA_FORMAT, Set.of(FULL_TEXT_SEARCH)));
134114
return idField;
135115
}
136116

‎sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LucenePluginBaseTests.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,12 @@
1111
import org.apache.lucene.document.FieldType;
1212
import org.apache.lucene.index.IndexOptions;
1313
import org.opensearch.be.lucene.LucenePlugin;
14+
import org.opensearch.common.lucene.Lucene;
1415
import org.opensearch.index.mapper.KeywordFieldMapper;
1516
import org.opensearch.index.mapper.MappedFieldType;
17+
import org.opensearch.index.mapper.MatchOnlyTextFieldMapper;
1618
import org.opensearch.index.mapper.TextFieldMapper;
19+
import org.opensearch.index.mapper.TextSearchInfo;
1720
import org.opensearch.test.OpenSearchTestCase;
1821

1922
import java.util.Map;
@@ -29,6 +32,18 @@ protected MappedFieldType mockTextField(String name) {
2932
return textFieldType;
3033
}
3134

35+
protected MappedFieldType mockMatchOnlyTextField(String name) {
36+
TextFieldMapper.TextFieldType textFieldType = new MatchOnlyTextFieldMapper.MatchOnlyTextFieldType(
37+
name,
38+
true,
39+
false,
40+
new TextSearchInfo(TextFieldMapper.Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER),
41+
Map.of()
42+
);
43+
textFieldType.setCapabilityMap(Map.of(LucenePlugin.DATA_FORMAT, Set.of(FULL_TEXT_SEARCH)));
44+
return textFieldType;
45+
}
46+
3247
protected MappedFieldType mockKeywordField(String name) {
3348
final FieldType keywordFieldType = new FieldType();
3449
keywordFieldType.setTokenized(false);

‎sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/AbstractSortedRefreshIT.java‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -67,15 +67,14 @@ public void tearDown() throws Exception {
6767
}
6868

6969
protected void createIndex(Settings settings) {
70-
client().admin()
71-
.indices()
72-
.prepareCreate(INDEX_NAME)
73-
.setSettings(settings)
74-
.setMapping("name", "type=keyword", "age", "type=integer", "tag", "type=keyword")
75-
.get();
70+
client().admin().indices().prepareCreate(INDEX_NAME).setSettings(settings).setMapping(getMapping()).get();
7671
ensureGreen(INDEX_NAME);
7772
}
7873

74+
protected String[] getMapping() {
75+
return new String[] { "name", "type=keyword", "age", "type=integer", "tag", "type=keyword" };
76+
}
77+
7978
protected void indexDoc(String name, int age) {
8079
IndexResponse response = client().prepareIndex().setIndex(INDEX_NAME).setSource("name", name, "age", age).get();
8180
assertEquals(RestStatus.CREATED, response.status());

‎sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeRefreshSortedParquetOnlyIT.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,4 +135,8 @@ private Settings unsortedParquetOnlySettings() {
135135
.build();
136136
}
137137

138+
@Override
139+
protected String[] getMapping() {
140+
return new String[] { "name", "type=keyword,index=false", "age", "type=integer", "tag", "type=keyword,index=false" };
141+
}
138142
}

‎sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/cancellation/AnalyticsQueryTaskCleanupIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
import org.opensearch.common.util.FeatureFlags;
2727
import org.opensearch.composite.CompositeDataFormatPlugin;
2828
import org.opensearch.index.engine.dataformat.stub.MockCommitterEnginePlugin;
29-
import org.opensearch.parquet.ParquetDataFormatPlugin;
29+
import org.opensearch.parquet.ParquetOnlyDataFormatPlugin;
3030
import org.opensearch.plugins.Plugin;
3131
import org.opensearch.plugins.PluginInfo;
3232
import org.opensearch.ppl.TestPPLPlugin;
@@ -96,7 +96,7 @@ protected Collection<PluginInfo> additionalNodePlugins() {
9696
return List.of(
9797
classpathPlugin(FlightStreamPlugin.class, List.of(ArrowBasePlugin.class.getName())),
9898
classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()),
99-
classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()),
99+
classpathPlugin(ParquetOnlyDataFormatPlugin.class, Collections.emptyList()),
100100
classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName()))
101101
);
102102
}

‎sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/cancellation/ReduceThreadPoolCleanupIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
import org.opensearch.common.util.FeatureFlags;
2727
import org.opensearch.composite.CompositeDataFormatPlugin;
2828
import org.opensearch.index.engine.dataformat.stub.MockCommitterEnginePlugin;
29-
import org.opensearch.parquet.ParquetDataFormatPlugin;
29+
import org.opensearch.parquet.ParquetOnlyDataFormatPlugin;
3030
import org.opensearch.plugins.Plugin;
3131
import org.opensearch.plugins.PluginInfo;
3232
import org.opensearch.ppl.TestPPLPlugin;
@@ -93,7 +93,7 @@ protected Collection<PluginInfo> additionalNodePlugins() {
9393
return List.of(
9494
classpathPlugin(FlightStreamPlugin.class, List.of(ArrowBasePlugin.class.getName())),
9595
classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()),
96-
classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()),
96+
classpathPlugin(ParquetOnlyDataFormatPlugin.class, Collections.emptyList()),
9797
classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName()))
9898
);
9999
}

‎sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/cancellation/SearchCancellationIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,14 @@
2828
import org.opensearch.common.unit.TimeValue;
2929
import org.opensearch.common.util.FeatureFlags;
3030
import org.opensearch.composite.CompositeDataFormatPlugin;
31-
import org.opensearch.parquet.ParquetDataFormatPlugin;
3231
import org.opensearch.plugins.Plugin;
3332
import org.opensearch.plugins.PluginInfo;
3433
import org.opensearch.ppl.TestPPLPlugin;
3534
import org.opensearch.ppl.action.PPLRequest;
3635
import org.opensearch.ppl.action.PPLResponse;
3736
import org.opensearch.ppl.action.UnifiedPPLExecuteAction;
3837
import org.opensearch.test.OpenSearchIntegTestCase;
38+
import org.opensearch.parquet.ParquetOnlyDataFormatPlugin;
3939
import org.opensearch.test.transport.MockTransportService;
4040
import org.opensearch.transport.TransportService;
4141

@@ -95,7 +95,7 @@ protected Collection<PluginInfo> additionalNodePlugins() {
9595
return List.of(
9696
classpathPlugin(FlightStreamPlugin.class, List.of(ArrowBasePlugin.class.getName())),
9797
classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()),
98-
classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()),
98+
classpathPlugin(ParquetOnlyDataFormatPlugin.class, Collections.emptyList()),
9999
classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName()))
100100
);
101101
}

‎sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/CoordinatorResilienceIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@
3939
import org.opensearch.common.util.FeatureFlags;
4040
import org.opensearch.composite.CompositeDataFormatPlugin;
4141
import org.opensearch.core.transport.TransportResponse;
42-
import org.opensearch.parquet.ParquetDataFormatPlugin;
42+
import org.opensearch.parquet.ParquetOnlyDataFormatPlugin;
4343
import org.opensearch.plugins.Plugin;
4444
import org.opensearch.plugins.PluginInfo;
4545
import org.opensearch.ppl.TestPPLPlugin;
@@ -157,7 +157,7 @@ protected Collection<PluginInfo> additionalNodePlugins() {
157157
return List.of(
158158
classpathPlugin(FlightStreamPlugin.class, List.of(ArrowBasePlugin.class.getName())),
159159
classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()),
160-
classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()),
160+
classpathPlugin(ParquetOnlyDataFormatPlugin.class, Collections.emptyList()),
161161
classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName()))
162162
);
163163
}

0 commit comments

Comments
 (0)