diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/BaseServiceIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/BaseServiceIT.java index d2faf59dc500..7a210bcdef2c 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/BaseServiceIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/BaseServiceIT.java @@ -200,8 +200,12 @@ void recursiveHardDelete_serviceSubtree_leavesNoOrphansAndSearchClean(TestNamesp subtree != null, "service type provides no deletable subtree builder; skipping"); for (SearchDoc sd : subtree.searchDocs()) { + // Secondary docs (e.g. column_search_index) are written on the async per-entity indexing lane + // and can sit briefly behind a concurrent full-reindex alias swap, so allow the same + // tolerance + // as the post-delete checks rather than a tighter 60s that flakes under full IT load. Awaitility.await("descendant indexed in search before delete: " + sd.index()) - .atMost(Duration.ofSeconds(60)) + .atMost(Duration.ofSeconds(120)) .pollInterval(Duration.ofSeconds(1)) .ignoreExceptions() .untilAsserted( diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/IndexingLimitsIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/IndexingLimitsIT.java new file mode 100644 index 000000000000..9d230d809547 --- /dev/null +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/IndexingLimitsIT.java @@ -0,0 +1,145 @@ +package org.openmetadata.it.tests; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import es.co.elastic.clients.transport.rest5_client.low_level.Request; +import es.co.elastic.clients.transport.rest5_client.low_level.Response; +import es.co.elastic.clients.transport.rest5_client.low_level.ResponseException; +import es.co.elastic.clients.transport.rest5_client.low_level.Rest5Client; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.stream.Stream; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.openmetadata.it.bootstrap.TestSuiteBootstrap; +import org.openmetadata.service.search.SearchFieldLimits; +import org.openmetadata.service.search.SearchIndexSettings; +import org.openmetadata.service.search.opensearch.OsUtils; + +/** + * Proves the engine-native hardening in {@link SearchIndexSettings} prevents documents from being + * rejected by the real search engine. For each risk class, a document is rejected by an index built + * from the raw mapping and accepted by an index built from the hardened mapping. Runs against + * whichever engine the active Maven profile selected, so the two CI profiles cover both engines. + */ +@Execution(ExecutionMode.CONCURRENT) +public class IndexingLimitsIT { + + private static final List CREATED_INDICES = new CopyOnWriteArrayList<>(); + + @AfterAll + static void cleanup() { + Rest5Client client = TestSuiteBootstrap.createSearchClient(); + for (String index : CREATED_INDICES) { + try { + client.performRequest(new Request("DELETE", "/" + index)); + } catch (Exception ignored) { + // best-effort cleanup + } + } + } + + @Test + void keywordOverByteLimitRejectedRawAcceptedWhenHardened() throws Exception { + String rawMapping = "{\"mappings\":{\"properties\":{\"fqn\":{\"type\":\"keyword\"}}}}"; + String doc = "{\"fqn\":\"" + "a".repeat(40000) + "\"}"; + + assertTrue( + rejects("kw_raw", rawMapping, doc), "raw keyword index must reject the immense term"); + assertTrue( + accepts("kw_hardened", harden(rawMapping), doc), + "hardened index (ignore_above) must accept the value"); + } + + static Stream malformedValuesByType() { + return Stream.of( + Arguments.of("integer", "\"not-a-number\""), + Arguments.of("long", "\"not-a-long\""), + Arguments.of("double", "\"not-a-double\""), + Arguments.of("float", "\"NaN\""), + Arguments.of("date", "\"not-a-date\""), + Arguments.of("boolean", "\"not-a-bool\"")); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("malformedValuesByType") + void malformedValueRejectedRawAcceptedWhenHardened(String type, String jsonValue) + throws Exception { + String rawMapping = "{\"mappings\":{\"properties\":{\"v\":{\"type\":\"" + type + "\"}}}}"; + String doc = "{\"v\":" + jsonValue + "}"; + + assertTrue( + rejects(type + "_raw", rawMapping, doc), + "raw " + type + " index must reject the bad value"); + // OpenSearch does not support ignore_malformed on boolean (OsUtils strips it), so a hardened + // boolean index there still rejects the malformed value; every other guarded type accepts on + // both engines. + boolean expectedToAccept = !("boolean".equals(type) && isOpenSearch()); + assertEquals( + expectedToAccept, + accepts(type + "_hardened", harden(rawMapping), doc), + "hardened " + type + " index acceptance must match the engine's ignore_malformed support"); + } + + private static boolean isOpenSearch() { + return "opensearch".equalsIgnoreCase(System.getProperty("searchType", "elasticsearch")); + } + + private String harden(String mapping) { + String hardened = SearchIndexSettings.harden(mapping, SearchFieldLimits.defaults()); + // Mirror the production OpenSearch path: harden() then enrichIndexMappingForOpenSearch() + // (e.g. strips ignore_malformed from boolean, which OpenSearch rejects). + if (isOpenSearch()) { + hardened = OsUtils.enrichIndexMappingForOpenSearch(hardened); + } + return hardened; + } + + private boolean rejects(String index, String mapping, String doc) throws Exception { + createIndex(index, mapping); + return indexStatus(index, doc) >= 400; + } + + private boolean accepts(String index, String mapping, String doc) throws Exception { + createIndex(index, mapping); + return indexStatus(index, doc) < 400; + } + + /** + * Status code of indexing {@code doc} into {@code index}. The rest5 low-level client surfaces a + * rejected write as the {@link Response} carrying the 4xx (not always a thrown exception), so the + * status code is authoritative — mirroring how {@code ElasticSearchClient} reads {@code + * e.getResponse()}. A thrown {@link ResponseException} is unwrapped to its response so a rejection + * is detected whichever way the client surfaces it. + */ + private int indexStatus(String index, String doc) throws Exception { + int statusCode; + try { + statusCode = index(index, doc).getStatusCode(); + } catch (ResponseException rejected) { + statusCode = rejected.getResponse().getStatusCode(); + } + return statusCode; + } + + private void createIndex(String index, String mapping) throws Exception { + Rest5Client client = TestSuiteBootstrap.createSearchClient(); + CREATED_INDICES.add(index); + Request request = new Request("PUT", "/" + index); + request.setJsonEntity(mapping); + client.performRequest(request); + } + + private Response index(String index, String doc) throws Exception { + Rest5Client client = TestSuiteBootstrap.createSearchClient(); + Request request = new Request("PUT", "/" + index + "/_doc/1?refresh=true"); + request.setJsonEntity(doc); + return client.performRequest(request); + } +} diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchIndexFieldLimitIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchIndexFieldLimitIT.java index d45e539aefc6..3be8ee61d14c 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchIndexFieldLimitIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchIndexFieldLimitIT.java @@ -102,12 +102,14 @@ static void cleanupCustomProperties() { } /** - * Test that the extension field uses flattened type in Elasticsearch mapping. - * With flattened type, all custom properties are stored in a single field, - * preventing field explosion. + * Test that the extension field is a disabled object in the mapping. Custom-property values can + * exceed Lucene's 32766-byte keyword limit, and flattened/flat_object leaves are indexed as + * keywords (which OpenSearch cannot guard with ignore_above), so the raw extension is stored but + * not indexed. Custom-property search goes through customPropertiesTyped instead. Storing it as a + * disabled object also prevents field explosion. */ @Test - void testExtensionFieldIsFlattenedType(TestNamespace ns) throws Exception { + void testExtensionFieldIsDisabledObject(TestNamespace ns) throws Exception { Rest5Client searchClient = TestSuiteBootstrap.createSearchClient(); Request request = new Request("GET", "/" + TABLE_INDEX + "/_mapping"); @@ -122,11 +124,10 @@ void testExtensionFieldIsFlattenedType(TestNamespace ns) throws Exception { JsonNode extensionMapping = findExtensionMapping(root); assertNotNull(extensionMapping, "Extension field should exist in mapping"); - String extensionType = extensionMapping.path("type").asText(); - assertTrue( - "flattened".equals(extensionType) || "flat_object".equals(extensionType), - "Extension field should be flattened (ES) or flat_object (OpenSearch) type, but was: " - + extensionType); + assertFalse( + extensionMapping.path("enabled").asBoolean(true), + "Extension field should be a disabled object (enabled:false), but was: " + + extensionMapping); } /** diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchIndexImmenseTermIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchIndexImmenseTermIT.java index d8fff7550b2e..b70ace8350b9 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchIndexImmenseTermIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchIndexImmenseTermIT.java @@ -13,8 +13,12 @@ package org.openmetadata.it.tests; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.net.URI; import java.net.URLEncoder; import java.net.http.HttpClient; @@ -23,6 +27,7 @@ import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.List; +import java.util.UUID; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.awaitility.Awaitility; @@ -47,71 +52,91 @@ * limit (32,766 bytes) — e.g. a ~50 KB Looker/DAX expression. * * - * - *

Not engine-gated: the original break reproduces on OpenSearch, and the test also asserts the - * deeply nested column name stays searchable via the analyzed {@code columnNamesFuzzy} field (built - * from the full child hierarchy), not the dropped flattened {@code children} field. */ @Slf4j @ExtendWith(TestNamespaceExtension.class) public class SearchIndexImmenseTermIT { private static final String CONTAINER_ASSET_TYPE = "container"; + private static final String MAPPING_LANGUAGE = "en"; private static final int LUCENE_MAX_TERM_BYTES = 32766; private static final int NESTING_DEPTH = 25; + private static final int RAISED_DEPTH_LIMIT = NESTING_DEPTH + 5; + private static final ObjectMapper MAPPER = new ObjectMapper(); private static final HttpClient HTTP_CLIENT = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10)).build(); + private record NestedContainer( + String containerId, String leafColumnName, String descriptionToken) {} + @Test - void oversizedLeafInDeeplyNestedColumnIndexesAndStaysSearchable(TestNamespace ns) - throws Exception { + void deepColumnSearchabilityFollowsConfiguredDepthLimit(TestNamespace ns) throws Exception { OpenMetadataClient client = SdkClients.adminClient(); - // Letters-only token: word_delimiter fragments alphanumeric runs on letter/digit boundaries, - // so a token carrying RUN_ID hex digits would tokenize inconsistently between the indexed - // column name and the query. Letters keep the leaf name a single, exactly-matchable term. - String token = ns.prefix("immense").replaceAll("[^a-zA-Z]", "").toLowerCase(); - String descriptionToken = "immensedesc" + token; + StorageService service = StorageServiceTestFactory.createS3(ns); + + // Default depth limit (20): the container indexes despite the >32 KB leaf, but the + // 25-level-deep + // column name is dropped from the flattened columnNamesFuzzy. + NestedContainer withDefault = createDeeplyNestedContainer(ns, client, service, "depthdefault"); + awaitContainerSearchable(withDefault.descriptionToken(), withDefault.containerId()); + assertFalse( + indexedColumnNamesFuzzy(withDefault).contains(withDefault.leafColumnName()), + "With the default depth limit the 25-level-deep column name must be dropped"); + + // Raise the container index depth limit past the nesting depth via the admin mappings API. + raiseContainerDepthLimit(RAISED_DEPTH_LIMIT); + + // A container indexed after the increase carries the deep column name in columnNamesFuzzy. + NestedContainer withRaised = createDeeplyNestedContainer(ns, client, service, "depthraised"); + awaitContainerSearchable(withRaised.descriptionToken(), withRaised.containerId()); + assertTrue( + indexedColumnNamesFuzzy(withRaised).contains(withRaised.leafColumnName()), + "After raising the depth limit the 25-level-deep column name must be indexed"); + } + + private NestedContainer createDeeplyNestedContainer( + TestNamespace ns, OpenMetadataClient client, StorageService service, String label) + throws Exception { + // Short, letters-only token: keeps the leaf name a single exactly-matchable term and keeps + // columnNamesFuzzy small enough to index and refresh promptly. + String token = + "imm" + + (UUID.randomUUID() + UUID.randomUUID().toString()) + .replaceAll("[^a-f]", "") + .substring(0, 10); + String descriptionToken = label + token; String leafColumnName = "deepleaf" + token; String oversizedLeaf = buildOversizedExpression(); assertTrue( oversizedLeaf.getBytes(StandardCharsets.UTF_8).length > LUCENE_MAX_TERM_BYTES, "Fixture must exceed the Lucene term limit to exercise the original failure"); - StorageService service = StorageServiceTestFactory.createS3(ns); Column topLevelColumn = buildDeeplyNestedColumn(token, leafColumnName, oversizedLeaf); ContainerDataModel dataModel = new ContainerDataModel().withColumns(List.of(topLevelColumn)); CreateContainer request = new CreateContainer(); - request.setName(ns.prefix("immense-container")); + request.setName(ns.prefix(label + "-container")); request.setService(service.getFullyQualifiedName()); request.setDescription(descriptionToken); request.setDataModel(dataModel); Container container = client.containers().create(request); - String containerId = container.getId().toString(); log.info( "Created container {} nested {} levels deep with a {}-byte leaf", - containerId, + container.getId(), NESTING_DEPTH, oversizedLeaf.getBytes(StandardCharsets.UTF_8).length); - - // Plain-object mapping would exceed the depth limit; flattened would fail the oversized leaf - // (immense term) on OpenSearch. object/enabled:false lets the document index in both respects. - awaitContainerSearchable(descriptionToken, containerId); - - HttpResponse byDeepColumn = searchContainers(leafColumnName); - assertEquals(200, byDeepColumn.statusCode(), "Deep nested column-name search must not error"); - assertTrue( - byDeepColumn.body().contains(containerId), - "A 25-level-deep column name must remain searchable via columnNamesFuzzy"); + return new NestedContainer(container.getId().toString(), leafColumnName, descriptionToken); } private Column buildDeeplyNestedColumn( @@ -143,17 +168,70 @@ private static String buildOversizedExpression() { return builder.toString(); } + /** + * Raises {@code settings.index.mapping.depth.limit} on the stored {@code container} mapping via the + * admin search-index-mappings API, leaving every other field intact. The change takes effect for + * containers indexed afterwards (the per-entity field limits are re-read once the setting cache is + * invalidated by the update). + */ + private void raiseContainerDepthLimit(int newDepthLimit) throws Exception { + HttpResponse current = + adminRequest( + "GET", + "/v1/system/settings/searchIndexMappings/" + + MAPPING_LANGUAGE + + "/" + + CONTAINER_ASSET_TYPE + + "?fallback=true", + null); + assertEquals(200, current.statusCode(), "Fetching the container mapping must succeed"); + + ObjectNode mapping = (ObjectNode) MAPPER.readTree(current.body()); + ObjectNode depth = + objectChild( + objectChild(objectChild(objectChild(mapping, "settings"), "index"), "mapping"), + "depth"); + depth.put("limit", newDepthLimit); + + HttpResponse updated = + adminRequest( + "PUT", + "/v1/system/settings/searchIndexMappings/" + + MAPPING_LANGUAGE + + "/" + + CONTAINER_ASSET_TYPE, + MAPPER.writeValueAsString(mapping)); + assertEquals(200, updated.statusCode(), "Updating the container mapping must succeed"); + } + + private static ObjectNode objectChild(ObjectNode parent, String field) { + ObjectNode result; + if (parent.get(field) instanceof ObjectNode existing) { + result = existing; + } else { + result = parent.putObject(field); + } + return result; + } + private void awaitContainerSearchable(String token, String containerId) { Awaitility.await("container indexed in search") .atMost(180, TimeUnit.SECONDS) - .pollInterval(1, TimeUnit.SECONDS) + .pollInterval(2, TimeUnit.SECONDS) .ignoreExceptions() - .untilAsserted( - () -> { - HttpResponse response = searchContainers(token); - assertEquals(200, response.statusCode()); - assertTrue(response.body().contains(containerId)); - }); + .untilAsserted(() -> assertTrue(searchContainers(token).body().contains(containerId))); + } + + /** The indexed {@code columnNamesFuzzy} of the container, located by its unique description. */ + private String indexedColumnNamesFuzzy(NestedContainer c) throws Exception { + JsonNode root = MAPPER.readTree(searchContainers(c.descriptionToken()).body()); + String result = ""; + for (JsonNode hit : root.path("hits").path("hits")) { + if (c.containerId().equals(hit.path("_source").path("id").asText())) { + result = hit.path("_source").path("columnNamesFuzzy").asText(); + } + } + return result; } private HttpResponse searchContainers(String token) throws Exception { @@ -163,15 +241,23 @@ private HttpResponse searchContainers(String token) throws Exception { + "&index=" + CONTAINER_ASSET_TYPE + "&from=0&size=10&deleted=false"; - HttpRequest httpRequest = + return adminRequest("GET", path, null); + } + + private HttpResponse adminRequest(String method, String path, String body) + throws Exception { + HttpRequest.Builder builder = HttpRequest.newBuilder() .uri(URI.create(SdkClients.getServerUrl() + path)) .header("Authorization", "Bearer " + SdkClients.getAdminToken()) .header("Content-Type", "application/json") .header("Accept", "application/json") - .timeout(Duration.ofSeconds(30)) - .GET() - .build(); - return HTTP_CLIENT.send(httpRequest, HttpResponse.BodyHandlers.ofString()); + .timeout(Duration.ofSeconds(30)); + if ("PUT".equals(method)) { + builder.PUT(HttpRequest.BodyPublishers.ofString(body)); + } else { + builder.GET(); + } + return HTTP_CLIENT.send(builder.build(), HttpResponse.BodyHandlers.ofString()); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 445c7d823d26..4918409128c8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -89,6 +89,7 @@ import org.openmetadata.schema.configuration.EntityRulesSettings; import org.openmetadata.schema.configuration.GlossaryTermRelationSettings; import org.openmetadata.schema.configuration.OpenLineageSettings; +import org.openmetadata.schema.configuration.SearchIndexMappings; import org.openmetadata.schema.configuration.WorkflowSettings; import org.openmetadata.schema.dataInsight.DataInsightChart; import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChart; @@ -10774,6 +10775,7 @@ public static Settings getSettings(SettingsType configType, String json) { case GLOSSARY_TERM_RELATION_SETTINGS -> JsonUtils.readValue( json, GlossaryTermRelationSettings.class); case AI_SETTINGS -> JsonUtils.readValue(json, AISettings.class); + case SEARCH_INDEX_MAPPINGS -> JsonUtils.readValue(json, SearchIndexMappings.class); default -> throw new IllegalArgumentException("Invalid Settings Type " + configType); }; settings.setConfigValue(value); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java index a4968c63dd47..e58a5a5d006a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java @@ -24,6 +24,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -55,6 +56,7 @@ import org.openmetadata.schema.configuration.LLMEmbeddingsConfig; import org.openmetadata.schema.configuration.LLMGoogleConfig; import org.openmetadata.schema.configuration.LLMOpenAIConfig; +import org.openmetadata.schema.configuration.SearchIndexMappings; import org.openmetadata.schema.configuration.SecurityConfiguration; import org.openmetadata.schema.configuration.WorkflowSettings; import org.openmetadata.schema.email.SmtpSettings; @@ -100,7 +102,10 @@ import org.openmetadata.service.resources.settings.SettingsCache; import org.openmetadata.service.search.IndexMappingVersionTracker; import org.openmetadata.service.search.IndexMappingVersionTracker.MappingDriftState; +import org.openmetadata.service.search.SearchFieldLimits; import org.openmetadata.service.search.SearchHealthStatus; +import org.openmetadata.service.search.SearchIndexMappingsSeeder; +import org.openmetadata.service.search.SearchIndexSettings; import org.openmetadata.service.search.SearchRepository; import org.openmetadata.service.search.vector.client.EmbeddingClient; import org.openmetadata.service.secrets.SecretsManager; @@ -143,7 +148,7 @@ private enum ValidationStepDescription { JWT_TOKEN("Validate that the ingestion-bot JWT token can be properly decoded."), MIGRATION("Validate that all the necessary migrations have been properly executed."), SEARCH_REINDEX( - "Validate that every deployed search index was built from the current code mapping " + "Validate that every deployed search index was built from the current index mapping " + "(i.e. no reindex is pending)."); public final String key; @@ -348,6 +353,84 @@ public Response deleteSettings(SettingsType type) { return (new RestUtil.DeleteResponse<>(oldValue, ENTITY_DELETED)).toResponse(); } + public SearchIndexMappings getSearchIndexMappings() { + SearchIndexMappings result = null; + Settings stored = getConfigWithKey(SettingsType.SEARCH_INDEX_MAPPINGS.toString()); + if (stored != null) { + result = JsonUtils.convertValue(stored.getConfigValue(), SearchIndexMappings.class); + } + return result; + } + + /** + * The stored mapping for a (language, entityType); when {@code fallbackToDefault} and no stored + * slice exists, the hardened resource default is returned instead. + */ + public Map getSearchIndexMapping( + String language, String entityType, boolean fallbackToDefault) { + Map result = storedSearchIndexMapping(language, entityType); + if (result == null && fallbackToDefault) { + result = SearchIndexMappingsSeeder.buildEntityMapping(language, entityType); + } + return result; + } + + public Settings upsertSearchIndexMapping( + String language, String entityType, Map mapping) { + return spliceSearchIndexMapping(language, entityType, hardenMapping(mapping)); + } + + public Settings resetSearchIndexMapping(String language, String entityType) { + Settings result = null; + Map defaultMapping = + SearchIndexMappingsSeeder.buildEntityMapping(language, entityType); + if (defaultMapping != null) { + result = spliceSearchIndexMapping(language, entityType, defaultMapping); + } + return result; + } + + private Settings spliceSearchIndexMapping( + String language, String entityType, Map mapping) { + SearchIndexMappings blob = getOrBuildSearchIndexMappings(); + blob.getLanguages() + .computeIfAbsent(language, key -> new LinkedHashMap<>()) + .put(entityType, mapping); + Settings setting = + new Settings().withConfigType(SettingsType.SEARCH_INDEX_MAPPINGS).withConfigValue(blob); + createOrUpdate(setting); + return setting; + } + + private SearchIndexMappings getOrBuildSearchIndexMappings() { + SearchIndexMappings blob = getSearchIndexMappings(); + if (blob == null) { + blob = new SearchIndexMappings(); + } + if (blob.getLanguages() == null) { + blob.setLanguages(new LinkedHashMap<>()); + } + return blob; + } + + private Map storedSearchIndexMapping(String language, String entityType) { + Map result = null; + SearchIndexMappings blob = getSearchIndexMappings(); + if (blob != null && blob.getLanguages() != null) { + Map byEntity = blob.getLanguages().get(language); + if (byEntity != null && byEntity.get(entityType) != null) { + result = JsonUtils.getMap(byEntity.get(entityType)); + } + } + return result; + } + + private Map hardenMapping(Map mapping) { + String hardened = + SearchIndexSettings.harden(JsonUtils.pojoToJson(mapping), SearchFieldLimits.active()); + return JsonUtils.getMapFromJson(hardened); + } + public Response patchSetting(String settingName, JsonPatch patch) { Settings original = getConfigWithKey(settingName); // Apply JSON patch to the original entity to get the updated entity @@ -1078,7 +1161,7 @@ private static void appendReindexNeeded(StringBuilder message, SearchReindexStat if (!status.stalePending().isEmpty()) { message.append( String.format( - " %d index(es) built from an older code mapping: %s.", + " %d index(es) built from an older mapping: %s.", status.stalePending().size(), status.stalePending())); } if (!status.missingIndexes().isEmpty()) { @@ -1090,7 +1173,7 @@ private static void appendReindexNeeded(StringBuilder message, SearchReindexStat } private static void appendUpToDate(StringBuilder message, int untrackedCount) { - message.append("All deployed indexes were built from the current code mappings."); + message.append("All deployed indexes were built from the current index mappings."); if (untrackedCount > 0) { message.append( String.format( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v200/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v200/Migration.java index fdfc22d9cd8c..bb9fc637eed7 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v200/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v200/Migration.java @@ -14,6 +14,7 @@ import org.openmetadata.service.migration.api.MigrationProcessImpl; import org.openmetadata.service.migration.utils.MigrationFile; import org.openmetadata.service.migration.utils.v200.MigrationUtil; +import org.openmetadata.service.search.SearchIndexMappingsSeeder; @Slf4j public class Migration extends MigrationProcessImpl { @@ -38,6 +39,7 @@ public void runDataMigration() { backfillAnnouncementRelationships(handle); addTaskAuthorPolicyToDataConsumerRole(collectionDAO); addCreateTaskRuleToDataConsumerPolicy(collectionDAO); + SearchIndexMappingsSeeder.seedIfAbsent(); // Wrap WorkflowHandler init + task workflow steps so a handler failure logs and continues // instead of aborting the rest of the v200 data migration. Matches v190/v171/v170/v1105. diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v200/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v200/Migration.java index e59ff7444643..7becf0574b14 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v200/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v200/Migration.java @@ -14,6 +14,7 @@ import org.openmetadata.service.migration.api.MigrationProcessImpl; import org.openmetadata.service.migration.utils.MigrationFile; import org.openmetadata.service.migration.utils.v200.MigrationUtil; +import org.openmetadata.service.search.SearchIndexMappingsSeeder; @Slf4j public class Migration extends MigrationProcessImpl { @@ -38,6 +39,7 @@ public void runDataMigration() { backfillAnnouncementRelationships(handle); addTaskAuthorPolicyToDataConsumerRole(collectionDAO); addCreateTaskRuleToDataConsumerPolicy(collectionDAO); + SearchIndexMappingsSeeder.seedIfAbsent(); // Wrap WorkflowHandler init + task workflow steps so a handler failure logs and continues // instead of aborting the rest of the v200 data migration. Matches v190/v171/v170/v1105. diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java index ba674129f37b..88d93d1036e8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java @@ -27,6 +27,7 @@ import static org.openmetadata.schema.settings.SettingsType.OPEN_LINEAGE_SETTINGS; import static org.openmetadata.schema.settings.SettingsType.OPEN_METADATA_BASE_URL_CONFIGURATION; import static org.openmetadata.schema.settings.SettingsType.SCIM_CONFIGURATION; +import static org.openmetadata.schema.settings.SettingsType.SEARCH_INDEX_MAPPINGS; import static org.openmetadata.schema.settings.SettingsType.SEARCH_SETTINGS; import static org.openmetadata.schema.settings.SettingsType.WORKFLOW_SETTINGS; @@ -77,6 +78,8 @@ import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.resources.system.AISettingsHandler; import org.openmetadata.service.resources.system.SearchSettingsHandler; +import org.openmetadata.service.search.SearchFieldLimits; +import org.openmetadata.service.search.SearchIndexMappingsSeeder; import org.openmetadata.service.search.SearchRepository; import org.openmetadata.service.search.indexes.SearchIndex; import org.openmetadata.service.util.EntityUtil; @@ -525,6 +528,9 @@ private static void createDefaultConfiguration(OpenMetadataApplicationConfig app new GlossaryTermRelationSettings().withRelationTypes(defaultRelationTypes)); Entity.getSystemRepository().createNewSetting(setting); } + + // Initialize admin-editable, per-language/per-entity search index mappings (hardened at seed) + SearchIndexMappingsSeeder.seedIfAbsent(); } private static GlossaryTermRelationType createRelationType( @@ -588,6 +594,10 @@ public static void invalidateSettings(String settingsName) { if (SEARCH_SETTINGS.toString().equals(settingsName)) { CACHE.invalidate(SEARCH_SETTINGS_AGGREGATED_FIELDS); } + // Stored mapping edits change the per-entity field limits (e.g. depth) used at build time + if (SEARCH_INDEX_MAPPINGS.toString().equals(settingsName)) { + SearchFieldLimits.invalidateEntityCache(); + } } catch (Exception ex) { LOG.error("Failed to invalidate cache for settings {}", settingsName, ex); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java index 2b8a1f223e28..c5ca33d43756 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java @@ -22,9 +22,11 @@ import jakarta.json.JsonPatch; import jakarta.json.JsonValue; import jakarta.validation.Valid; +import jakarta.ws.rs.BadRequestException; import jakarta.ws.rs.Consumes; import jakarta.ws.rs.DefaultValue; import jakarta.ws.rs.GET; +import jakarta.ws.rs.NotFoundException; import jakarta.ws.rs.PATCH; import jakarta.ws.rs.POST; import jakarta.ws.rs.PUT; @@ -40,6 +42,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.UUID; import lombok.extern.slf4j.Slf4j; @@ -89,6 +92,7 @@ import org.openmetadata.service.resources.Collection; import org.openmetadata.service.resources.settings.SettingsCache; import org.openmetadata.service.rules.LogicOps; +import org.openmetadata.service.search.SearchIndexMappingsSeeder; import org.openmetadata.service.secrets.masker.PasswordEntityMasker; import org.openmetadata.service.security.Authorizer; import org.openmetadata.service.security.JwtFilter; @@ -110,6 +114,8 @@ @LatencyPhase public class SystemResource { public static final String COLLECTION_PATH = "/v1/system"; + private static final String MAPPINGS_KEY = "mappings"; + private static final String PROPERTIES_KEY = "properties"; private final SystemRepository systemRepository; private final Authorizer authorizer; private OpenMetadataApplicationConfig applicationConfig; @@ -299,6 +305,112 @@ public List getEntityRulesSettingByType( .toList(); } + @GET + @Path("/settings/searchIndexMappings") + @Operation( + operationId = "listSearchIndexMappings", + summary = "List editable search index mappings", + description = + "List every editable entity type, grouped by search index mapping language. Entities " + + "without a saved override resolve from the bundled default mapping.") + public Map> listSearchIndexMappings( + @Context SecurityContext securityContext) { + authorizer.authorizeAdmin(securityContext); + return SearchIndexMappingsSeeder.availableMappings(); + } + + @GET + @Path("/settings/searchIndexMappings/{language}/{entityType}") + @Operation( + operationId = "getSearchIndexMapping", + summary = "Get the search index mapping for an entity in a language", + description = + "Get the stored, editable index mapping for an entity type and language. When " + + "'fallback' is true, returns the hardened default mapping if none is stored yet.") + public Map getSearchIndexMapping( + @Context SecurityContext securityContext, + @PathParam("language") String language, + @PathParam("entityType") String entityType, + @QueryParam("fallback") @DefaultValue("false") boolean fallback) { + authorizer.authorizeAdmin(securityContext); + Map mapping = + systemRepository.getSearchIndexMapping( + language.toLowerCase(Locale.ROOT), entityType, fallback); + if (mapping == null) { + throw new NotFoundException( + String.format( + "No stored search index mapping for language '%s' and entity '%s'", + language, entityType)); + } + return mapping; + } + + @PUT + @Path("/settings/searchIndexMappings/{language}/{entityType}") + @Operation( + operationId = "updateSearchIndexMapping", + summary = "Update the search index mapping for an entity in a language", + description = + "Persist an admin-edited index mapping for an entity type and language. The submitted " + + "mapping is field-safety hardened before storage. The change takes effect on the " + + "next reindex of that entity.") + public Response updateSearchIndexMapping( + @Context SecurityContext securityContext, + @PathParam("language") String language, + @PathParam("entityType") String entityType, + Map mapping) { + authorizer.authorizeAdmin(securityContext); + validateSearchIndexMappingRequest(language, entityType, mapping); + Settings updated = + systemRepository.upsertSearchIndexMapping( + language.toLowerCase(Locale.ROOT), entityType, mapping); + return Response.ok(updated).build(); + } + + @PUT + @Path("/settings/searchIndexMappings/reset/{language}/{entityType}") + @Operation( + operationId = "resetSearchIndexMapping", + summary = "Reset the search index mapping for an entity to its default", + description = + "Replace the stored mapping for an entity type and language with the hardened default " + + "derived from the bundled resource mapping. Applies on the next reindex.") + public Response resetSearchIndexMapping( + @Context SecurityContext securityContext, + @PathParam("language") String language, + @PathParam("entityType") String entityType) { + authorizer.authorizeAdmin(securityContext); + Settings reset = + systemRepository.resetSearchIndexMapping(language.toLowerCase(Locale.ROOT), entityType); + if (reset == null) { + throw new NotFoundException( + String.format("No default search index mapping for entity '%s'", entityType)); + } + return Response.ok(reset).build(); + } + + private void validateSearchIndexMappingRequest( + String language, String entityType, Map mapping) { + if (!SearchIndexMappingsSeeder.supportedLanguages() + .contains(language.toLowerCase(Locale.ROOT))) { + throw new BadRequestException("Unsupported search index mapping language: " + language); + } + if (!SearchIndexMappingsSeeder.supportedEntityTypes().contains(entityType)) { + throw new BadRequestException("Unknown search index entity type: " + entityType); + } + if (!hasMappingProperties(mapping)) { + throw new BadRequestException("Index mapping must contain a 'mappings.properties' object"); + } + } + + private boolean hasMappingProperties(Map mapping) { + boolean valid = false; + if (mapping != null && mapping.get(MAPPINGS_KEY) instanceof Map mappings) { + valid = mappings.get(PROPERTIES_KEY) instanceof Map; + } + return valid; + } + @GET @Path("/search/nlq") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/CustomPropertySearchFields.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/CustomPropertySearchFields.java new file mode 100644 index 000000000000..cced3bc25133 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/CustomPropertySearchFields.java @@ -0,0 +1,89 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.search; + +import java.util.ArrayList; +import java.util.List; +import org.openmetadata.schema.api.search.AssetTypeConfiguration; +import org.openmetadata.schema.api.search.FieldBoost; + +/** + * Resolves an admin-configured custom-property search field (stored as {@code extension.}) into + * the nested {@code customPropertiesTyped} sub-fields that actually hold the indexed value. + * + *

The raw {@code extension} object is mapped {@code enabled:false} (stored in {@code _source}, + * never indexed) so an oversized custom-property value can never reject the whole entity document — + * OpenSearch maps {@code extension} as {@code flat_object}, which cannot be guarded with {@code + * ignore_above}. The searchable, engine-safe copy of every custom property lives in the {@code + * customPropertiesTyped} nested field (property {@code name} plus a typed value), whose keyword + * sub-fields {@link SearchIndexSettings#harden} guards with {@code ignore_above}. So a search setting + * on {@code extension.} is served at query time by a nested query over {@code + * customPropertiesTyped} matching {@code name == } and the searched value — the query builders + * translate the configured field, the stored setting keeps the {@code extension.} identifier. + */ +public final class CustomPropertySearchFields { + + public static final String CUSTOM_PROPERTIES_TYPED = "customPropertiesTyped"; + public static final String NAME_FIELD = CUSTOM_PROPERTIES_TYPED + ".name"; + public static final String TEXT_VALUE_FIELD = CUSTOM_PROPERTIES_TYPED + ".textValue"; + public static final String STRING_VALUE_FIELD = CUSTOM_PROPERTIES_TYPED + ".stringValue"; + + private static final String EXTENSION_PREFIX = "extension."; + private static final String MATCH_TYPE_EXACT = "exact"; + private static final float DEFAULT_BOOST = 1.0f; + + private CustomPropertySearchFields() {} + + /** + * A resolved custom-property search field: the property name (the part after {@code extension.}), + * its configured boost, and whether the admin configured an exact-match strategy. + */ + public record Spec(String propertyName, float boost, boolean exact) {} + + /** The custom-property search fields configured on {@code assetConfig}, in configuration order. */ + public static List from(AssetTypeConfiguration assetConfig) { + List specs = new ArrayList<>(); + if (assetConfig != null && assetConfig.getSearchFields() != null) { + for (FieldBoost fieldBoost : assetConfig.getSearchFields()) { + Spec spec = toSpec(fieldBoost); + if (spec != null) { + specs.add(spec); + } + } + } + return specs; + } + + private static Spec toSpec(FieldBoost fieldBoost) { + Spec spec = null; + String field = fieldBoost.getField(); + if (isCustomPropertyField(field)) { + String propertyName = field.substring(EXTENSION_PREFIX.length()); + float boost = + fieldBoost.getBoost() != null ? fieldBoost.getBoost().floatValue() : DEFAULT_BOOST; + spec = new Spec(propertyName, boost, isExactMatch(fieldBoost)); + } + return spec; + } + + private static boolean isCustomPropertyField(String field) { + return field != null + && field.startsWith(EXTENSION_PREFIX) + && field.length() > EXTENSION_PREFIX.length(); + } + + private static boolean isExactMatch(FieldBoost fieldBoost) { + return fieldBoost.getMatchType() != null + && MATCH_TYPE_EXACT.equals(fieldBoost.getMatchType().value()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/IndexMappingVersionTracker.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/IndexMappingVersionTracker.java index 5f97f216071f..2f670c0159c0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/IndexMappingVersionTracker.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/IndexMappingVersionTracker.java @@ -15,10 +15,11 @@ import java.util.Map; import java.util.Objects; import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.configuration.SearchIndexMappings; import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.schema.utils.VersionUtils; -import org.openmetadata.search.IndexMapping; import org.openmetadata.search.IndexMappingLoader; +import org.openmetadata.service.Entity; import org.openmetadata.service.exception.IndexMappingHashException; import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.IndexMappingVersionDAO; @@ -204,36 +205,45 @@ private Map getStoredMappingHashes() { private record MappingEntry(String hash, JsonNode json) {} + /** + * Drift is computed against the effective mapping each index is built from: the + * admin-editable mapping stored in the {@code searchIndexMappings} setting, falling back to the + * hardened bundled default when a slice is absent. So an admin edit (or a code default change) + * surfaces as a reindex-required drift until the entity is reindexed. + */ private Map computeCurrentMappings() throws IOException { Map mappings = new HashMap<>(); - - // Use IndexMappingLoader as the source of truth for entity types and their mapping file paths. - // This avoids constructing file paths manually and ensures all entity types are covered, - // including camelCase ones like glossaryTerm, databaseSchema, etc. - Map indexMappings = IndexMappingLoader.getInstance().getIndexMapping(); - - for (Map.Entry entry : indexMappings.entrySet()) { - MappingEntry mappingEntry = toMappingEntry(entry.getKey(), entry.getValue()); + SearchIndexMappings storedBlob = loadStoredMappings(); + for (String entityType : IndexMappingLoader.getInstance().getIndexMapping().keySet()) { + MappingEntry mappingEntry = toMappingEntry(entityType, storedBlob); if (mappingEntry != null) { - mappings.put(entry.getKey(), mappingEntry); + mappings.put(entityType, mappingEntry); } } - return mappings; } private MappingEntry computeMappingForEntity(String entityType) throws IOException { - IndexMapping indexMapping = IndexMappingLoader.getInstance().getIndexMapping().get(entityType); MappingEntry result = null; - if (indexMapping != null) { - result = toMappingEntry(entityType, indexMapping); + if (IndexMappingLoader.getInstance().getIndexMapping().containsKey(entityType)) { + result = toMappingEntry(entityType, loadStoredMappings()); } return result; } - private MappingEntry toMappingEntry(String entityType, IndexMapping indexMapping) + private SearchIndexMappings loadStoredMappings() { + SearchIndexMappings blob = null; + try { + blob = Entity.getSystemRepository().getSearchIndexMappings(); + } catch (Exception e) { + LOG.debug("Could not load stored search index mappings; using bundled defaults", e); + } + return blob; + } + + private MappingEntry toMappingEntry(String entityType, SearchIndexMappings storedBlob) throws IOException { - JsonNode mapping = loadMappingForEntity(entityType, indexMapping); + JsonNode mapping = loadEffectiveMapping(entityType, storedBlob); MappingEntry result = null; if (mapping != null) { try { @@ -246,29 +256,44 @@ private MappingEntry toMappingEntry(String entityType, IndexMapping indexMapping return result; } - private JsonNode loadMappingForEntity(String entityType, IndexMapping indexMapping) { + private JsonNode loadEffectiveMapping(String entityType, SearchIndexMappings storedBlob) { + JsonNode result = null; try { Map allLanguageMappings = new HashMap<>(); - String[] languages = {"en", "jp", "ru", "zh"}; - - for (String lang : languages) { - // Use the indexMappingFile from indexMapping.json which has the correct path template - String mappingPath = "/" + indexMapping.getIndexMappingFile(lang); - try (var stream = getClass().getResourceAsStream(mappingPath)) { - if (stream != null) { - String mappingContent = new String(stream.readAllBytes(), StandardCharsets.UTF_8); - allLanguageMappings.put(lang, MAPPER.readTree(mappingContent)); - } + for (String language : SearchIndexMappingsSeeder.supportedLanguages()) { + Object mapping = effectiveMapping(entityType, language, storedBlob); + if (mapping != null) { + allLanguageMappings.put(language, MAPPER.valueToTree(mapping)); } } - if (!allLanguageMappings.isEmpty()) { - return MAPPER.valueToTree(allLanguageMappings); + result = MAPPER.valueToTree(allLanguageMappings); } } catch (Exception e) { LOG.debug("Could not load mapping for entity: {}", entityType, e); } - return null; + return result; + } + + private Object effectiveMapping( + String entityType, String language, SearchIndexMappings storedBlob) { + Object mapping = storedMappingSlice(storedBlob, language, entityType); + if (mapping == null) { + mapping = SearchIndexMappingsSeeder.buildEntityMapping(language, entityType); + } + return mapping; + } + + private Object storedMappingSlice( + SearchIndexMappings storedBlob, String language, String entityType) { + Object slice = null; + if (storedBlob != null && storedBlob.getLanguages() != null) { + Map byEntity = storedBlob.getLanguages().get(language); + if (byEntity != null) { + slice = byEntity.get(entityType); + } + } + return slice; } private String computeHash(JsonNode mapping) throws IOException, IndexMappingHashException { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchFieldLimits.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchFieldLimits.java new file mode 100644 index 000000000000..39592c385666 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchFieldLimits.java @@ -0,0 +1,259 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.search; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.openmetadata.schema.configuration.SearchIndexMappings; +import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration; +import org.openmetadata.schema.service.configuration.elasticsearch.SearchIndexingLimits; +import org.openmetadata.schema.settings.SettingsType; +import org.openmetadata.search.IndexMappingLoader; +import org.openmetadata.service.resources.settings.SettingsCache; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Immutable holder for the field limits enforced while building search documents. Values default to + * the documented Elasticsearch/OpenSearch engine defaults and can be overridden via {@link + * SearchIndexingLimits} in the search configuration so operators can tune them without changing + * cluster/infrastructure settings. + */ +public final class SearchFieldLimits { + + private static final Logger LOG = LoggerFactory.getLogger(SearchFieldLimits.class); + + private static volatile SearchFieldLimits active; + + /** + * Per-entity-type limits resolved from the stored, admin-editable mapping (its {@code + * settings.index.mapping.depth.limit}), so that updating a mapping via the search-index-mappings + * API and reindexing changes the document-build depth for that entity. Memoized to avoid + * deserializing the mapping blob on every document build; cleared by {@link #invalidateEntityCache} + * when the stored mappings change. Bounded by the number of entity types. + */ + private static final Map PER_ENTITY = new ConcurrentHashMap<>(); + + private static final List DEPTH_LIMIT_PATH = + List.of("settings", "index", "mapping", "depth", "limit"); + + /** Hard Lucene per-term limit ({@code IndexWriter.MAX_TERM_LENGTH}); not configurable upward. */ + public static final int LUCENE_KEYWORD_MAX_BYTES = 32766; + + public static final int DEFAULT_DEPTH_LIMIT = 20; + public static final int DEFAULT_NESTED_OBJECTS_LIMIT = 10000; + public static final int DEFAULT_TOTAL_FIELDS_LIMIT = 1000; + public static final int DEFAULT_MAX_COLUMNS = 10000; + + private static final int MAX_UTF8_BYTES_PER_CHAR = 4; + + private final boolean hardeningEnabled; + private final int keywordMaxBytes; + private final int depthLimit; + private final int nestedObjectsLimit; + private final int totalFieldsLimit; + private final int maxColumns; + private final int safeCharThreshold; + + private SearchFieldLimits( + boolean hardeningEnabled, + int keywordMaxBytes, + int depthLimit, + int nestedObjectsLimit, + int totalFieldsLimit, + int maxColumns) { + this.hardeningEnabled = hardeningEnabled; + this.keywordMaxBytes = keywordMaxBytes; + this.depthLimit = depthLimit; + this.nestedObjectsLimit = nestedObjectsLimit; + this.totalFieldsLimit = totalFieldsLimit; + this.maxColumns = maxColumns; + this.safeCharThreshold = keywordMaxBytes / MAX_UTF8_BYTES_PER_CHAR; + } + + public static SearchFieldLimits defaults() { + return new SearchFieldLimits( + true, + LUCENE_KEYWORD_MAX_BYTES, + DEFAULT_DEPTH_LIMIT, + DEFAULT_NESTED_OBJECTS_LIMIT, + DEFAULT_TOTAL_FIELDS_LIMIT, + DEFAULT_MAX_COLUMNS); + } + + public static SearchFieldLimits from(ElasticSearchConfiguration configuration) { + SearchFieldLimits result = defaults(); + if (configuration != null && configuration.getSearchIndexingLimits() != null) { + result = fromLimits(configuration.getSearchIndexingLimits()); + } + return result; + } + + /** + * The limits resolved from the running search configuration, cached after first use. Falls back to + * {@link #defaults()} when the configuration has not been initialized (e.g. in unit tests). + */ + public static SearchFieldLimits active() { + SearchFieldLimits result = active; + if (result == null) { + result = loadActive(); + } + return result; + } + + public static void setActive(SearchFieldLimits limits) { + active = limits; + PER_ENTITY.clear(); + } + + /** + * Limits for building documents of {@code entityType}, overlaying the depth limit from that + * entity's stored mapping onto {@link #active()}. Falls back to {@link #active()} when no entity + * type is given or no stored override exists. + */ + public static SearchFieldLimits forEntity(String entityType) { + SearchFieldLimits result = active(); + if (entityType != null && !entityType.isBlank()) { + result = PER_ENTITY.computeIfAbsent(entityType, SearchFieldLimits::resolveForEntity); + } + return result; + } + + /** Drops the memoized per-entity limits so the next build re-reads the stored mappings. */ + public static void invalidateEntityCache() { + PER_ENTITY.clear(); + } + + private static SearchFieldLimits resolveForEntity(String entityType) { + SearchFieldLimits base = active(); + Integer storedDepth = storedDepthLimit(entityType); + SearchFieldLimits result = base; + if (storedDepth != null && storedDepth > 0 && storedDepth != base.getDepthLimit()) { + result = base.withDepthLimit(storedDepth); + } + return result; + } + + private static Integer storedDepthLimit(String entityType) { + Integer result = null; + try { + SearchIndexMappings stored = + SettingsCache.getSettingOrDefault( + SettingsType.SEARCH_INDEX_MAPPINGS, null, SearchIndexMappings.class); + result = depthLimitFromMapping(entityMapping(stored, entityType)); + } catch (Exception notResolved) { + LOG.debug("Could not resolve stored depth limit for entity {}", entityType, notResolved); + } + return result; + } + + @SuppressWarnings("unchecked") + private static Map entityMapping(SearchIndexMappings stored, String entityType) { + Map result = null; + if (stored != null && stored.getLanguages() != null) { + for (Map byEntity : stored.getLanguages().values()) { + if (byEntity != null && byEntity.get(entityType) instanceof Map mapping) { + result = (Map) mapping; + break; + } + } + } + return result; + } + + @SuppressWarnings("unchecked") + private static Integer depthLimitFromMapping(Map mapping) { + Object node = mapping; + for (String key : DEPTH_LIMIT_PATH) { + node = node instanceof Map ? ((Map) node).get(key) : null; + } + return node instanceof Number depth ? depth.intValue() : null; + } + + SearchFieldLimits withDepthLimit(int newDepthLimit) { + return new SearchFieldLimits( + hardeningEnabled, + keywordMaxBytes, + newDepthLimit, + nestedObjectsLimit, + totalFieldsLimit, + maxColumns); + } + + private static synchronized SearchFieldLimits loadActive() { + SearchFieldLimits result = active; + if (result == null) { + try { + result = from(IndexMappingLoader.getInstance().getElasticSearchConfiguration()); + active = result; + } catch (IllegalStateException notInitialized) { + LOG.debug("IndexMappingLoader not initialized; using default search field limits"); + result = defaults(); + } + } + return result; + } + + private static SearchFieldLimits fromLimits(SearchIndexingLimits limits) { + return new SearchFieldLimits( + limits.getEnableMappingHardening() == null || limits.getEnableMappingHardening(), + clampKeywordBytes(limits.getKeywordMaxBytes()), + orDefault(limits.getMappingDepthLimit(), DEFAULT_DEPTH_LIMIT), + orDefault(limits.getNestedObjectsLimit(), DEFAULT_NESTED_OBJECTS_LIMIT), + orDefault(limits.getTotalFieldsLimit(), DEFAULT_TOTAL_FIELDS_LIMIT), + orDefault(limits.getMaxColumns(), DEFAULT_MAX_COLUMNS)); + } + + private static int orDefault(Integer value, int fallback) { + return value != null && value > 0 ? value : fallback; + } + + private static int clampKeywordBytes(Integer value) { + int resolved = orDefault(value, LUCENE_KEYWORD_MAX_BYTES); + // Keep at least one UTF-8 character's worth of bytes so ignore_above (value/4) is never 0. + return Math.min(Math.max(resolved, MAX_UTF8_BYTES_PER_CHAR), LUCENE_KEYWORD_MAX_BYTES); + } + + public boolean isHardeningEnabled() { + return hardeningEnabled; + } + + public int getKeywordMaxBytes() { + return keywordMaxBytes; + } + + public int getDepthLimit() { + return depthLimit; + } + + public int getNestedObjectsLimit() { + return nestedObjectsLimit; + } + + public int getTotalFieldsLimit() { + return totalFieldsLimit; + } + + public int getMaxColumns() { + return maxColumns; + } + + /** + * Strings at or below this character count cannot exceed {@link #getKeywordMaxBytes()} bytes (UTF-8 + * is at most 4 bytes per character), so callers can skip byte-length computation for them. + */ + public int getSafeCharThreshold() { + return safeCharThreshold; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexMappingsSeeder.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexMappingsSeeder.java new file mode 100644 index 000000000000..d1753b48b99c --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexMappingsSeeder.java @@ -0,0 +1,195 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.search; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import org.openmetadata.schema.configuration.SearchIndexMappings; +import org.openmetadata.schema.settings.Settings; +import org.openmetadata.schema.settings.SettingsType; +import org.openmetadata.schema.type.IndexMappingLanguage; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.search.IndexMapping; +import org.openmetadata.search.IndexMappingLoader; +import org.openmetadata.service.Entity; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Resolves the field-safety-hardened search index mappings used at index-creation time. The {@code + * searchIndexMappings} setting stores only admin overrides — entities the operator has + * explicitly edited — so an un-edited entity always resolves from the current bundled resource + * (hardened on the fly by {@link SearchIndexSettings#harden}). That means shipped mapping changes + * take effect automatically on the next reindex, while admin edits are preserved. Shared by the + * per-entity reset endpoint and the runtime resource fallback. {@link #buildDefaultBlob} remains for + * building a full hardened snapshot on demand (e.g. tooling/tests). + */ +public final class SearchIndexMappingsSeeder { + + private static final Logger LOG = LoggerFactory.getLogger(SearchIndexMappingsSeeder.class); + + private SearchIndexMappingsSeeder() {} + + /** + * Initializes an empty override store for search index mappings if the setting is absent. + * Defaults are intentionally not pre-seeded — each (language, entityType) resolves from the + * bundled hardened resource unless an admin saved an override — so a later upgrade's mapping + * changes apply without a manual reset. Insert-if-absent, never clobbers existing overrides. + * Failures are logged, not propagated, so a hiccup never aborts startup or a migration. + */ + public static void seedIfAbsent() { + try { + if (Entity.getSystemRepository() + .getConfigWithKey(SettingsType.SEARCH_INDEX_MAPPINGS.toString()) + == null) { + Settings setting = + new Settings() + .withConfigType(SettingsType.SEARCH_INDEX_MAPPINGS) + .withConfigValue(new SearchIndexMappings().withLanguages(new LinkedHashMap<>())); + Entity.getSystemRepository().createNewSetting(setting); + LOG.info("Initialized empty search index mappings override store"); + } + } catch (Exception seedFailed) { + LOG.error("Failed to initialize search index mappings setting", seedFailed); + } + } + + public static List supportedLanguages() { + return Arrays.stream(IndexMappingLanguage.values()) + .map(language -> language.value().toLowerCase(Locale.ROOT)) + .toList(); + } + + /** Entity types that have a bundled index mapping and are therefore editable. */ + public static List supportedEntityTypes() { + ensureLoaderInitialized(); + return IndexMappingLoader.getInstance().getIndexMapping().keySet().stream().sorted().toList(); + } + + /** + * The editable (language → entity type) matrix, sourced from the bundled mappings rather than the + * stored overrides, so every editable entity is listed even before any override is saved. + */ + public static Map> availableMappings() { + List entityTypes = supportedEntityTypes(); + Map> result = new LinkedHashMap<>(); + for (String language : supportedLanguages()) { + result.put(language, entityTypes); + } + return result; + } + + public static SearchIndexMappings buildDefaultBlob() { + return buildDefaultBlob(supportedLanguages()); + } + + public static SearchIndexMappings buildDefaultBlob(List languages) { + ensureLoaderInitialized(); + SearchFieldLimits limits = SearchFieldLimits.active(); + Map registry = IndexMappingLoader.getInstance().getIndexMapping(); + Map> byLanguage = new LinkedHashMap<>(); + for (String language : languages) { + byLanguage.put(language, buildLanguageMappings(registry, language, limits)); + } + return new SearchIndexMappings().withLanguages(byLanguage); + } + + /** Hardened default mapping for one (language, entityType), or {@code null} when none exists. */ + public static Map buildEntityMapping(String language, String entityType) { + Map result = null; + // Resolve to the enum's own value so the request-supplied language never composes the classpath + // resource path directly (path-injection barrier); an unknown language yields no mapping. + String safeLanguage = canonicalLanguage(language); + if (safeLanguage != null) { + ensureLoaderInitialized(); + IndexMapping indexMapping = + IndexMappingLoader.getInstance().getIndexMapping().get(entityType); + if (indexMapping != null) { + result = hardenedResourceMapping(indexMapping, safeLanguage, SearchFieldLimits.active()); + } + } + return result; + } + + /** + * Maps a request-supplied language to the matching {@link IndexMappingLanguage} constant's own + * value, or {@code null} when unrecognized. The returned string originates from the enum, not the + * request, so callers can use it to build resource paths without carrying user-controlled taint. + */ + private static String canonicalLanguage(String language) { + String result = null; + for (IndexMappingLanguage candidate : IndexMappingLanguage.values()) { + if (candidate.value().equalsIgnoreCase(language)) { + result = candidate.value().toLowerCase(Locale.ROOT); + break; + } + } + return result; + } + + private static Map buildLanguageMappings( + Map registry, String language, SearchFieldLimits limits) { + Map entityMappings = new LinkedHashMap<>(); + for (Map.Entry entry : registry.entrySet()) { + Map mapping = hardenedResourceMapping(entry.getValue(), language, limits); + if (mapping != null) { + entityMappings.put(entry.getKey(), mapping); + } + } + return entityMappings; + } + + private static Map hardenedResourceMapping( + IndexMapping indexMapping, String language, SearchFieldLimits limits) { + Map result = null; + String content = readResource(indexMapping.getIndexMappingFile(language)); + if (content != null) { + result = JsonUtils.getMapFromJson(SearchIndexSettings.harden(content, limits)); + } + return result; + } + + private static String readResource(String resourcePath) { + String result = null; + try (InputStream in = + SearchIndexMappingsSeeder.class.getClassLoader().getResourceAsStream(resourcePath)) { + if (in == null) { + LOG.debug("No index mapping resource at {}", resourcePath); + } else { + result = new String(in.readAllBytes(), StandardCharsets.UTF_8); + } + } catch (IOException readFailed) { + LOG.warn("Failed reading index mapping resource {}", resourcePath, readFailed); + } + return result; + } + + private static void ensureLoaderInitialized() { + try { + IndexMappingLoader.getInstance(); + } catch (IllegalStateException notInitialized) { + try { + IndexMappingLoader.init(); + } catch (IOException initFailed) { + throw new IllegalStateException( + "Failed to initialize IndexMappingLoader for search index mapping seeding", initFailed); + } + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexSettings.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexSettings.java new file mode 100644 index 000000000000..8078e8d834ee --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexSettings.java @@ -0,0 +1,187 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.search; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.util.Set; +import org.openmetadata.schema.utils.JsonUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Hardens an index's create-time mapping JSON so documents can never be rejected at index time, + * without any per-document work — Elasticsearch/OpenSearch enforce the bounds natively. Applied once + * per index creation, it: + * + *

+ * + * Existing values in a mapping file are never overwritten. + */ +public final class SearchIndexSettings { + + private static final Logger LOG = LoggerFactory.getLogger(SearchIndexSettings.class); + + private static final String LIMIT = "limit"; + private static final String TYPE = "type"; + private static final String PROPERTIES = "properties"; + private static final String FIELDS = "fields"; + private static final String IGNORE_ABOVE = "ignore_above"; + private static final String IGNORE_MALFORMED = "ignore_malformed"; + private static final String DEPTH_LIMIT = "depth_limit"; + private static final String KEYWORD = "keyword"; + private static final String FLATTENED = "flattened"; + + private static final Set MALFORMED_GUARD_TYPES = + Set.of( + "long", + "integer", + "short", + "byte", + "double", + "float", + "half_float", + "scaled_float", + "date", + "boolean"); + + private SearchIndexSettings() {} + + public static String harden(String indexMappingContent, SearchFieldLimits limits) { + String result = indexMappingContent; + if (limits.isHardeningEnabled() + && indexMappingContent != null + && !indexMappingContent.isEmpty()) { + result = hardenContent(indexMappingContent, limits); + } + return result; + } + + private static String hardenContent(String content, SearchFieldLimits limits) { + String result = content; + try { + ObjectNode root = (ObjectNode) JsonUtils.readTree(content); + injectMappingLimits(root, limits); + hardenFields(root, limits); + result = JsonUtils.pojoToJson(root); + } catch (Exception hardeningFailed) { + LOG.warn("Could not harden index mapping; using original", hardeningFailed); + } + return result; + } + + private static void hardenFields(ObjectNode root, SearchFieldLimits limits) { + JsonNode mappings = root.get("mappings"); + if (mappings instanceof ObjectNode mappingsNode + && mappingsNode.get(PROPERTIES) instanceof ObjectNode properties) { + hardenProperties(properties, limits); + } + } + + private static void hardenProperties(ObjectNode properties, SearchFieldLimits limits) { + properties + .fields() + .forEachRemaining( + entry -> { + if (entry.getValue() instanceof ObjectNode field) { + hardenField(field, limits); + } + }); + } + + private static void hardenField(ObjectNode field, SearchFieldLimits limits) { + String type = field.path(TYPE).asText(""); + applyKeywordGuard(field, type, limits); + applyMalformedGuard(field, type); + applyFlattenedGuard(field, type, limits); + if (field.get(PROPERTIES) instanceof ObjectNode nestedProperties) { + hardenProperties(nestedProperties, limits); + } + if (field.get(FIELDS) instanceof ObjectNode multiFields) { + hardenProperties(multiFields, limits); + } + } + + private static void applyKeywordGuard(ObjectNode field, String type, SearchFieldLimits limits) { + if (KEYWORD.equals(type) && !field.has(IGNORE_ABOVE)) { + field.put(IGNORE_ABOVE, limits.getSafeCharThreshold()); + } + } + + /** + * Elasticsearch {@code flattened} indexes every leaf as a keyword, so a long leaf would throw an + * immense-term error and a deep object would exceed the flattened depth limit. {@code ignore_above} + * and {@code depth_limit} guard both. These are ES-only parameters; the OpenSearch transform strips + * them when converting to {@code flat_object}. + */ + private static void applyFlattenedGuard(ObjectNode field, String type, SearchFieldLimits limits) { + if (FLATTENED.equals(type)) { + if (!field.has(IGNORE_ABOVE)) { + field.put(IGNORE_ABOVE, limits.getSafeCharThreshold()); + } + if (!field.has(DEPTH_LIMIT)) { + field.put(DEPTH_LIMIT, limits.getDepthLimit()); + } + } + } + + private static void applyMalformedGuard(ObjectNode field, String type) { + if (MALFORMED_GUARD_TYPES.contains(type) && !field.has(IGNORE_MALFORMED)) { + field.put(IGNORE_MALFORMED, true); + } + } + + private static void injectMappingLimits(ObjectNode root, SearchFieldLimits limits) { + ObjectNode mapping = mappingNode(root); + putLimitIfAbsent(mapping, "depth", limits.getDepthLimit()); + putLimitIfAbsent(mapping, "nested_objects", limits.getNestedObjectsLimit()); + putLimitIfAbsent(mapping, "total_fields", limits.getTotalFieldsLimit()); + } + + private static ObjectNode mappingNode(ObjectNode root) { + ObjectNode settings = childObject(root, "settings"); + ObjectNode index = childObject(settings, "index"); + return childObject(index, "mapping"); + } + + private static ObjectNode childObject(ObjectNode parent, String name) { + JsonNode existing = parent.get(name); + ObjectNode result; + if (existing != null && existing.isObject()) { + result = (ObjectNode) existing; + } else { + result = JsonUtils.getObjectNode(); + parent.set(name, result); + } + return result; + } + + private static void putLimitIfAbsent(ObjectNode mapping, String key, int value) { + JsonNode existing = mapping.get(key); + if (existing == null || !existing.has(LIMIT)) { + ObjectNode limitNode = JsonUtils.getObjectNode(); + limitNode.put(LIMIT, value); + mapping.set(key, limitNode); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index 60d5151b8335..9326840df1b1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -73,6 +73,7 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -102,6 +103,7 @@ import org.openmetadata.schema.configuration.AssetCertificationSettings; import org.openmetadata.schema.configuration.LLMConfiguration; import org.openmetadata.schema.configuration.LLMEmbeddingsConfig; +import org.openmetadata.schema.configuration.SearchIndexMappings; import org.openmetadata.schema.dataInsight.DataInsightChartResult; import org.openmetadata.schema.entity.classification.Tag; import org.openmetadata.schema.entity.data.Pipeline; @@ -923,25 +925,74 @@ public void deleteIndex(IndexMapping indexMapping) { } } - private String getIndexMapping(IndexMapping indexMapping) { + /** + * The effective index mapping for an entity. Prefers the admin-editable mapping persisted in the + * {@code searchIndexMappings} setting (already field-safety hardened at seed time); falls back to + * the hardened classpath resource when no stored slice exists (e.g. fresh-install first boot, or a + * newly added entity type not yet seeded). + */ + public String readIndexMapping(IndexMapping indexMapping) { + String mapping = getStoredMapping(indexMapping); + if (mapping == null) { + mapping = getHardenedResourceMapping(indexMapping); + } + if (isVectorEmbeddingEnabled() && embeddingClient != null && mapping != null) { + mapping = reformatVectorIndexWithDimension(mapping, embeddingClient.getDimension()); + } + return mapping; + } + + private String getStoredMapping(IndexMapping indexMapping) { + String result = null; + String entityType = resolveEntityType(indexMapping); + if (entityType != null) { + Object mapping = lookupStoredMapping(language.toLowerCase(Locale.ROOT), entityType); + if (mapping != null) { + result = JsonUtils.pojoToJson(mapping); + } + } + return result; + } + + private Object lookupStoredMapping(String mappingLanguage, String entityType) { + Object result = null; + SearchIndexMappings stored = + SettingsCache.getSettingOrDefault( + SettingsType.SEARCH_INDEX_MAPPINGS, null, SearchIndexMappings.class); + if (stored != null && stored.getLanguages() != null) { + Map byEntity = stored.getLanguages().get(mappingLanguage); + if (byEntity != null) { + result = byEntity.get(entityType); + } + } + return result; + } + + private String resolveEntityType(IndexMapping indexMapping) { + return entityIndexMap == null || indexMapping == null + ? null + : entityIndexMap.entrySet().stream() + .filter(entry -> entry.getValue().getIndexName().equals(indexMapping.getIndexName())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(null); + } + + private String getHardenedResourceMapping(IndexMapping indexMapping) { + String result = null; try (InputStream in = getClass() .getResourceAsStream( - String.format(indexMapping.getIndexMappingFile(), language.toLowerCase()))) { - assert in != null; - return new String(in.readAllBytes()); + String.format( + indexMapping.getIndexMappingFile(), language.toLowerCase(Locale.ROOT)))) { + if (in != null) { + result = + SearchIndexSettings.harden(new String(in.readAllBytes()), SearchFieldLimits.active()); + } } catch (Exception e) { LOG.error("Failed to read index Mapping file due to ", e); } - return null; - } - - public String readIndexMapping(IndexMapping indexMapping) { - String mapping = getIndexMapping(indexMapping); - if (isVectorEmbeddingEnabled() && embeddingClient != null && mapping != null) { - mapping = reformatVectorIndexWithDimension(mapping, embeddingClient.getDimension()); - } - return mapping; + return result; } /** diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchSourceBuilderFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchSourceBuilderFactory.java index ee62799dd9f2..02580f995859 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchSourceBuilderFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchSourceBuilderFactory.java @@ -29,6 +29,7 @@ import org.openmetadata.schema.api.search.SearchSettings; import org.openmetadata.schema.api.search.TermBoost; import org.openmetadata.service.Entity; +import org.openmetadata.service.search.CustomPropertySearchFields; import org.openmetadata.service.search.SearchSourceBuilderFactory; import org.openmetadata.service.search.indexes.ColumnSearchIndex; import org.openmetadata.service.search.indexes.SearchIndex; @@ -597,11 +598,69 @@ private es.co.elastic.clients.elasticsearch._types.query_dsl.Query buildSimpleQu addFuzzyMatchQueriesV2( combinedQuery, query, fieldsByMatchType.get(MATCH_TYPE_FUZZY), multipliers.fuzzyMatch); addStandardMatchQueriesV2(combinedQuery, query, fieldsByMatchType.get(MATCH_TYPE_STANDARD)); + addCustomPropertyMatchQueriesV2(combinedQuery, query, assetConfig); combinedQuery.minimumShouldMatch(1); return ElasticQueryBuilder.boolQuery().must(combinedQuery.build()).build(); } + /** + * Adds a nested {@code customPropertiesTyped} clause for each admin-configured {@code + * extension.} search field. The raw {@code extension} field is {@code enabled:false} so an + * oversized value can never reject the document, so the searchable value lives in the typed nested + * field — see {@link CustomPropertySearchFields}. + */ + private void addCustomPropertyMatchQueriesV2( + ElasticQueryBuilder.BoolQueryBuilder combinedQuery, + String query, + AssetTypeConfiguration assetConfig) { + for (CustomPropertySearchFields.Spec spec : CustomPropertySearchFields.from(assetConfig)) { + combinedQuery.should(customPropertyNestedQueryV2(query, spec)); + } + } + + private es.co.elastic.clients.elasticsearch._types.query_dsl.Query customPropertyNestedQueryV2( + String query, CustomPropertySearchFields.Spec spec) { + es.co.elastic.clients.elasticsearch._types.query_dsl.Query inner = + ElasticQueryBuilder.boolQuery() + .must(customPropertyTermV2(CustomPropertySearchFields.NAME_FIELD, spec.propertyName())) + .must(customPropertyValueQueryV2(query, spec)) + .build(); + return ElasticQueryBuilder.nestedQuery( + CustomPropertySearchFields.CUSTOM_PROPERTIES_TYPED, inner); + } + + private es.co.elastic.clients.elasticsearch._types.query_dsl.Query customPropertyValueQueryV2( + String query, CustomPropertySearchFields.Spec spec) { + es.co.elastic.clients.elasticsearch._types.query_dsl.Query result; + if (spec.exact()) { + result = + es.co.elastic.clients.elasticsearch._types.query_dsl.Query.of( + q -> + q.term( + t -> + t.field(CustomPropertySearchFields.STRING_VALUE_FIELD) + .value(query) + .boost(spec.boost()))); + } else { + result = + es.co.elastic.clients.elasticsearch._types.query_dsl.Query.of( + q -> + q.match( + m -> + m.field(CustomPropertySearchFields.TEXT_VALUE_FIELD) + .query(query) + .boost(spec.boost()))); + } + return result; + } + + private static es.co.elastic.clients.elasticsearch._types.query_dsl.Query customPropertyTermV2( + String field, String value) { + return es.co.elastic.clients.elasticsearch._types.query_dsl.Query.of( + q -> q.term(t -> t.field(field).value(value))); + } + private void addExactMatchQueriesV2( ElasticQueryBuilder.BoolQueryBuilder combinedQuery, String query, @@ -983,6 +1042,7 @@ private es.co.elastic.clients.elasticsearch._types.query_dsl.Query buildSimpleQu Map> fieldsByType = groupFieldsByMatchType(assetConfig); addMatchTypeQueriesV2(combinedQuery, query, fieldsByType, multipliers); + addCustomPropertyMatchQueriesV2(combinedQuery, query, assetConfig); combinedQuery.minimumShouldMatch(1); return ElasticQueryBuilder.boolQuery().must(combinedQuery.build()).build(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/APIEndpointIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/APIEndpointIndex.java index cdac466ece18..d6833a686b79 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/APIEndpointIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/APIEndpointIndex.java @@ -4,15 +4,11 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; -import java.util.function.Predicate; import org.openmetadata.schema.entity.data.APIEndpoint; -import org.openmetadata.schema.type.Field; import org.openmetadata.schema.type.TagLabel; import org.openmetadata.service.Entity; import org.openmetadata.service.search.models.FlattenSchemaField; -import org.openmetadata.service.util.FullyQualifiedName; public class APIEndpointIndex implements DataAssetIndex { final Set excludeAPIEndpointFields = Set.of("sampleData"); @@ -54,7 +50,8 @@ public Map buildSearchIndexDocInternal(Map doc) && apiEndpoint.getResponseSchema().getSchemaFields() != null && !apiEndpoint.getResponseSchema().getSchemaFields().isEmpty()) { List flattenFields = new ArrayList<>(); - parseSchemaFields(apiEndpoint.getResponseSchema().getSchemaFields(), flattenFields, null); + SchemaFieldFlattener.parseSchemaFields( + apiEndpoint.getResponseSchema().getSchemaFields(), flattenFields, null); List fieldsWithChildrenName = new ArrayList<>(); for (FlattenSchemaField field : flattenFields) { @@ -71,7 +68,8 @@ public Map buildSearchIndexDocInternal(Map doc) && apiEndpoint.getRequestSchema().getSchemaFields() != null && !apiEndpoint.getRequestSchema().getSchemaFields().isEmpty()) { List flattenFields = new ArrayList<>(); - parseSchemaFields(apiEndpoint.getRequestSchema().getSchemaFields(), flattenFields, null); + SchemaFieldFlattener.parseSchemaFields( + apiEndpoint.getRequestSchema().getSchemaFields(), flattenFields, null); List fieldsWithChildrenName = new ArrayList<>(); for (FlattenSchemaField field : flattenFields) { @@ -95,33 +93,6 @@ public Map buildSearchIndexDocInternal(Map doc) return doc; } - private void parseSchemaFields( - List fields, List flattenSchemaFields, String parentSchemaField) { - Optional optParentField = - Optional.ofNullable(parentSchemaField).filter(Predicate.not(String::isEmpty)); - List tags = new ArrayList<>(); - for (Field field : fields) { - String fieldName = field.getName(); - if (optParentField.isPresent()) { - fieldName = FullyQualifiedName.add(optParentField.get(), fieldName); - } - if (field.getTags() != null) { - tags = field.getTags(); - } - - FlattenSchemaField flattenSchemaField = - FlattenSchemaField.builder().name(fieldName).description(field.getDescription()).build(); - - if (!tags.isEmpty()) { - flattenSchemaField.setTags(tags); - } - flattenSchemaFields.add(flattenSchemaField); - if (field.getChildren() != null) { - parseSchemaFields(field.getChildren(), flattenSchemaFields, field.getName()); - } - } - } - public static Map getFields() { Map fields = SearchIndex.getDefaultFields(); fields.put("requestSchema.schemaFields.name.keyword", 5.0f); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ColumnIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ColumnIndex.java index fd40d921ce03..64e30ce669cb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ColumnIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ColumnIndex.java @@ -10,35 +10,78 @@ import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.type.Column; import org.openmetadata.schema.type.TagLabel; +import org.openmetadata.service.search.SearchFieldLimits; import org.openmetadata.service.search.models.FlattenColumn; import org.openmetadata.service.util.FullyQualifiedName; public interface ColumnIndex extends SearchIndex { default void parseColumns( List columns, List flattenColumns, String parentColumn) { + parseColumns( + columns, flattenColumns, parentColumn, 1, SearchFieldLimits.forEntity(getEntityTypeName())); + } + + private void parseColumns( + List columns, + List flattenColumns, + String parentColumn, + int depth, + SearchFieldLimits limits) { + if (depth > limits.getDepthLimit()) { + LOG.warn( + "Dropping columns under '{}' beyond mapping depth limit {}", + parentColumn, + limits.getDepthLimit()); + } else { + addColumnsWithinLimit(columns, flattenColumns, parentColumn, depth, limits); + } + } + + private void addColumnsWithinLimit( + List columns, + List flattenColumns, + String parentColumn, + int depth, + SearchFieldLimits limits) { Optional optParentColumn = Optional.ofNullable(parentColumn).filter(Predicate.not(String::isEmpty)); - List tags = new ArrayList<>(); - for (Column col : columns) { - String columnName = col.getName(); - if (optParentColumn.isPresent()) { - columnName = FullyQualifiedName.add(optParentColumn.get(), columnName); - } - if (col.getTags() != null) { - tags = col.getTags(); + int index = 0; + boolean capReached = false; + while (index < columns.size() && !capReached) { + if (flattenColumns.size() >= limits.getMaxColumns()) { + LOG.warn( + "Reached max indexed columns {}; dropping remaining columns under '{}'", + limits.getMaxColumns(), + parentColumn); + capReached = true; + } else { + Column col = columns.get(index); + List tags = col.getTags() != null ? col.getTags() : new ArrayList<>(); + String columnName = addFlattenColumn(col, optParentColumn, tags, flattenColumns); + if (col.getChildren() != null) { + parseColumns(col.getChildren(), flattenColumns, columnName, depth + 1, limits); + } + index++; } + } + } - FlattenColumn flattenColumn = - FlattenColumn.builder().name(columnName).description(col.getDescription()).build(); - - if (!tags.isEmpty()) { - flattenColumn.setTags(tags); - } - flattenColumns.add(flattenColumn); - if (col.getChildren() != null) { - parseColumns(col.getChildren(), flattenColumns, col.getName()); - } + private String addFlattenColumn( + Column col, + Optional optParentColumn, + List tags, + List flattenColumns) { + String columnName = col.getName(); + if (optParentColumn.isPresent()) { + columnName = FullyQualifiedName.add(optParentColumn.get(), columnName); + } + FlattenColumn flattenColumn = + FlattenColumn.builder().name(columnName).description(col.getDescription()).build(); + if (!tags.isEmpty()) { + flattenColumn.setTags(tags); } + flattenColumns.add(flattenColumn); + return columnName; } default String getColumnDescriptionStatus(EntityInterface entity) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ColumnSearchIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ColumnSearchIndex.java index 6ab6deee288d..e9a15ec7e34c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ColumnSearchIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ColumnSearchIndex.java @@ -14,6 +14,7 @@ import org.openmetadata.schema.type.TagLabel; import org.openmetadata.service.Entity; import org.openmetadata.service.search.ParseTags; +import org.openmetadata.service.search.SearchFieldLimits; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.util.FullyQualifiedName; @@ -170,15 +171,39 @@ public static Map getFields() { public static List flattenColumns(List columns) { List result = new ArrayList<>(); - if (columns == null) { - return result; + flattenColumns(columns, result, 1, SearchFieldLimits.forEntity(Entity.TABLE)); + return result; + } + + private static void flattenColumns( + List columns, List result, int depth, SearchFieldLimits limits) { + if (columns != null && depth > limits.getDepthLimit()) { + SearchIndex.LOG.warn( + "Dropping columns beyond mapping depth limit {} during column index flatten", + limits.getDepthLimit()); + } else if (columns != null) { + addFlattenedColumns(columns, result, depth, limits); } - for (Column col : columns) { - result.add(col); - if (col.getChildren() != null && !col.getChildren().isEmpty()) { - result.addAll(flattenColumns(col.getChildren())); + } + + private static void addFlattenedColumns( + List columns, List result, int depth, SearchFieldLimits limits) { + int index = 0; + boolean capReached = false; + while (index < columns.size() && !capReached) { + if (result.size() >= limits.getMaxColumns()) { + SearchIndex.LOG.warn( + "Reached max indexed columns {}; dropping remaining during column index flatten", + limits.getMaxColumns()); + capReached = true; + } else { + Column col = columns.get(index); + result.add(col); + if (col.getChildren() != null && !col.getChildren().isEmpty()) { + flattenColumns(col.getChildren(), result, depth + 1, limits); + } + index++; } } - return result; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/IngestionPipelineIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/IngestionPipelineIndex.java index d3f9a59f6874..1f79a916c929 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/IngestionPipelineIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/IngestionPipelineIndex.java @@ -1,12 +1,15 @@ package org.openmetadata.service.search.indexes; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import org.json.JSONObject; import org.openmetadata.schema.entity.services.ingestionPipelines.AirflowConfig; import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; +import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineStatus; import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.service.Entity; @@ -14,6 +17,12 @@ public class IngestionPipelineIndex implements TaggableIndex, ServiceBackedIndex final IngestionPipeline ingestionPipeline; final Set excludeFields = Set.of("sourceConfig", "openMetadataServerConnection"); + /** + * Free-form per-run map fields on a {@link PipelineStatus} that are not searched and whose + * arbitrary keys must never reach the index (see {@link #searchableStatus}). + */ + private static final Set NON_SEARCHABLE_STATUS_FIELDS = Set.of("config", "metadata"); + public IngestionPipelineIndex(IngestionPipeline ingestionPipeline) { this.ingestionPipeline = ingestionPipeline; } @@ -46,7 +55,7 @@ public Map buildSearchIndexDocInternal(Map doc) ingestionPipeline.getName() != null ? ingestionPipeline.getName() : ingestionPipeline.getDisplayName()); - doc.put("pipelineStatuses", ingestionPipeline.getPipelineStatuses()); + doc.put("pipelineStatuses", searchableStatuses(ingestionPipeline.getPipelineStatuses())); Optional.ofNullable(ingestionPipeline.getAirflowConfig()) .map(AirflowConfig::getScheduleInterval) .ifPresent( @@ -63,6 +72,35 @@ public Map buildSearchIndexDocInternal(Map doc) return doc; } + private List> searchableStatuses(List statuses) { + List> result = null; + if (statuses != null) { + result = new ArrayList<>(statuses.size()); + for (PipelineStatus status : statuses) { + result.add(searchableStatus(status)); + } + } + return result; + } + + /** + * Drops the free-form {@code config} and {@code metadata} blobs from the run status before + * indexing. Both are arbitrary per-run key/value maps (a {@code map} in the schema): they are not + * searched (only the derived {@code applicationType} is), can be large, and dynamically mapping + * their arbitrary keys previously triggered type conflicts (a key seen first as a string then as an + * object) and can push the index past its total-fields limit — either of which rejects the whole + * document. The searchable status fields (state, runId, timestamps) are preserved. Returns a copy + * so the entity is not mutated. + */ + private Map searchableStatus(PipelineStatus status) { + Map result = null; + if (status != null) { + result = JsonUtils.getMap(status); + result.keySet().removeAll(NON_SEARCHABLE_STATUS_FIELDS); + } + return result; + } + public static Map getFields() { return SearchIndex.getDefaultFields(); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SchemaFieldFlattener.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SchemaFieldFlattener.java new file mode 100644 index 000000000000..9ee54c127488 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SchemaFieldFlattener.java @@ -0,0 +1,106 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.search.indexes; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.function.Predicate; +import org.openmetadata.schema.type.Field; +import org.openmetadata.schema.type.TagLabel; +import org.openmetadata.service.search.SearchFieldLimits; +import org.openmetadata.service.search.models.FlattenSchemaField; +import org.openmetadata.service.util.FullyQualifiedName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Flattens recursive message/API schema fields (Topic {@code messageSchema}, API endpoint + * request/response schemas) into a flat list, bounded by the configured depth and field-count limits + * so a pathologically deep or wide schema can never explode the search document. Shared by the topic + * and API-endpoint indexes, mirroring the column flatten cap in {@link ColumnIndex}. + */ +public final class SchemaFieldFlattener { + + private static final Logger LOG = LoggerFactory.getLogger(SchemaFieldFlattener.class); + + private SchemaFieldFlattener() {} + + public static void parseSchemaFields( + List fields, List flattenSchemaFields, String parentField) { + parseSchemaFields(fields, flattenSchemaFields, parentField, 1, SearchFieldLimits.active()); + } + + private static void parseSchemaFields( + List fields, + List flattenSchemaFields, + String parentField, + int depth, + SearchFieldLimits limits) { + if (depth > limits.getDepthLimit()) { + LOG.warn( + "Dropping schema fields under '{}' beyond mapping depth limit {}", + parentField, + limits.getDepthLimit()); + } else { + addFieldsWithinLimit(fields, flattenSchemaFields, parentField, depth, limits); + } + } + + private static void addFieldsWithinLimit( + List fields, + List flattenSchemaFields, + String parentField, + int depth, + SearchFieldLimits limits) { + Optional optParentField = + Optional.ofNullable(parentField).filter(Predicate.not(String::isEmpty)); + int index = 0; + boolean capReached = false; + while (index < fields.size() && !capReached) { + if (flattenSchemaFields.size() >= limits.getMaxColumns()) { + LOG.warn( + "Reached max indexed schema fields {}; dropping remaining under '{}'", + limits.getMaxColumns(), + parentField); + capReached = true; + } else { + Field field = fields.get(index); + List tags = field.getTags() != null ? field.getTags() : new ArrayList<>(); + String fieldName = addFlattenField(field, optParentField, tags, flattenSchemaFields); + if (field.getChildren() != null) { + parseSchemaFields(field.getChildren(), flattenSchemaFields, fieldName, depth + 1, limits); + } + index++; + } + } + } + + private static String addFlattenField( + Field field, + Optional optParentField, + List tags, + List flattenSchemaFields) { + String fieldName = field.getName(); + if (optParentField.isPresent()) { + fieldName = FullyQualifiedName.add(optParentField.get(), fieldName); + } + FlattenSchemaField flattenSchemaField = + FlattenSchemaField.builder().name(fieldName).description(field.getDescription()).build(); + if (!tags.isEmpty()) { + flattenSchemaField.setTags(tags); + } + flattenSchemaFields.add(flattenSchemaField); + return fieldName; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java index 4effdc969b09..ec667ca5f0fb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java @@ -6,15 +6,11 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; -import java.util.function.Predicate; import org.openmetadata.schema.entity.data.Topic; -import org.openmetadata.schema.type.Field; import org.openmetadata.schema.type.TagLabel; import org.openmetadata.service.Entity; import org.openmetadata.service.search.models.FlattenSchemaField; -import org.openmetadata.service.util.FullyQualifiedName; public class TopicIndex implements DataAssetIndex { final Set excludeTopicFields = Set.of("sampleData"); @@ -58,7 +54,8 @@ public Map buildSearchIndexDocInternal(Map doc) && topic.getMessageSchema().getSchemaFields() != null && !topic.getMessageSchema().getSchemaFields().isEmpty()) { List flattenFields = new ArrayList<>(); - parseSchemaFields(topic.getMessageSchema().getSchemaFields(), flattenFields, null); + SchemaFieldFlattener.parseSchemaFields( + topic.getMessageSchema().getSchemaFields(), flattenFields, null); List fieldsWithChildrenName = new ArrayList<>(); Set> childTags = new HashSet<>(); @@ -77,33 +74,6 @@ public Map buildSearchIndexDocInternal(Map doc) return doc; } - private void parseSchemaFields( - List fields, List flattenSchemaFields, String parentSchemaField) { - Optional optParentField = - Optional.ofNullable(parentSchemaField).filter(Predicate.not(String::isEmpty)); - List tags = new ArrayList<>(); - for (Field field : fields) { - String fieldName = field.getName(); - if (optParentField.isPresent()) { - fieldName = FullyQualifiedName.add(optParentField.get(), fieldName); - } - if (field.getTags() != null) { - tags = field.getTags(); - } - - FlattenSchemaField flattenSchemaField = - FlattenSchemaField.builder().name(fieldName).description(field.getDescription()).build(); - - if (!tags.isEmpty()) { - flattenSchemaField.setTags(tags); - } - flattenSchemaFields.add(flattenSchemaField); - if (field.getChildren() != null) { - parseSchemaFields(field.getChildren(), flattenSchemaFields, field.getName()); - } - } - } - public static Map getFields() { Map fields = SearchIndex.getDefaultFields(); fields.put(ES_MESSAGE_SCHEMA_FIELD, 7.0f); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchSourceBuilderFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchSourceBuilderFactory.java index c71c61faa34e..88ecffa8d203 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchSourceBuilderFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchSourceBuilderFactory.java @@ -26,6 +26,7 @@ import org.openmetadata.schema.api.search.SearchSettings; import org.openmetadata.schema.api.search.TermBoost; import org.openmetadata.service.Entity; +import org.openmetadata.service.search.CustomPropertySearchFields; import org.openmetadata.service.search.SearchSourceBuilderFactory; import org.openmetadata.service.search.indexes.ColumnSearchIndex; import org.openmetadata.service.search.indexes.SearchIndex; @@ -511,6 +512,7 @@ private os.org.opensearch.client.opensearch._types.query_dsl.Query buildSimpleQu Map> fieldsByType = groupFieldsByMatchType(assetConfig); addMatchTypeQueriesV2(combinedQuery, query, fieldsByType, multipliers); + addCustomPropertyMatchQueriesV2(combinedQuery, query, assetConfig); combinedQuery.minimumShouldMatch(1); return OpenSearchQueryBuilder.boolQuery().must(combinedQuery.build()).build(); @@ -1075,11 +1077,69 @@ private os.org.opensearch.client.opensearch._types.query_dsl.Query buildSimpleQu addFuzzyMatchQueriesV2( combinedQuery, query, fieldsByMatchType.get(MATCH_TYPE_FUZZY), multipliers.fuzzyMatch); addStandardMatchQueriesV2(combinedQuery, query, fieldsByMatchType.get(MATCH_TYPE_STANDARD)); + addCustomPropertyMatchQueriesV2(combinedQuery, query, assetConfig); combinedQuery.minimumShouldMatch(1); return OpenSearchQueryBuilder.boolQuery().must(combinedQuery.build()).build(); } + /** + * Adds a nested {@code customPropertiesTyped} clause for each admin-configured {@code + * extension.} search field. The raw {@code extension} field is {@code enabled:false} so an + * oversized value can never reject the document, so the searchable value lives in the typed nested + * field — see {@link CustomPropertySearchFields}. + */ + private void addCustomPropertyMatchQueriesV2( + OpenSearchQueryBuilder.BoolQueryBuilder combinedQuery, + String query, + AssetTypeConfiguration assetConfig) { + for (CustomPropertySearchFields.Spec spec : CustomPropertySearchFields.from(assetConfig)) { + combinedQuery.should(customPropertyNestedQueryV2(query, spec)); + } + } + + private os.org.opensearch.client.opensearch._types.query_dsl.Query customPropertyNestedQueryV2( + String query, CustomPropertySearchFields.Spec spec) { + os.org.opensearch.client.opensearch._types.query_dsl.Query inner = + OpenSearchQueryBuilder.boolQuery() + .must(customPropertyTermV2(CustomPropertySearchFields.NAME_FIELD, spec.propertyName())) + .must(customPropertyValueQueryV2(query, spec)) + .build(); + return OpenSearchQueryBuilder.nestedQuery( + CustomPropertySearchFields.CUSTOM_PROPERTIES_TYPED, inner); + } + + private os.org.opensearch.client.opensearch._types.query_dsl.Query customPropertyValueQueryV2( + String query, CustomPropertySearchFields.Spec spec) { + os.org.opensearch.client.opensearch._types.query_dsl.Query result; + if (spec.exact()) { + result = + os.org.opensearch.client.opensearch._types.query_dsl.Query.of( + q -> + q.term( + t -> + t.field(CustomPropertySearchFields.STRING_VALUE_FIELD) + .value(FieldValue.of(query)) + .boost(spec.boost()))); + } else { + result = + os.org.opensearch.client.opensearch._types.query_dsl.Query.of( + q -> + q.match( + m -> + m.field(CustomPropertySearchFields.TEXT_VALUE_FIELD) + .query(FieldValue.of(query)) + .boost(spec.boost()))); + } + return result; + } + + private static os.org.opensearch.client.opensearch._types.query_dsl.Query customPropertyTermV2( + String field, String value) { + return os.org.opensearch.client.opensearch._types.query_dsl.Query.of( + q -> q.term(t -> t.field(field).value(FieldValue.of(value)))); + } + private void addExactMatchQueriesV2( OpenSearchQueryBuilder.BoolQueryBuilder combinedQuery, String query, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OsUtils.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OsUtils.java index 4082bdd0aad5..7ed87047f8fb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OsUtils.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OsUtils.java @@ -635,9 +635,16 @@ private static void transformFieldTypesRecursive(ObjectNode propertiesNode) { JsonNode typeNode = fieldObj.get("type"); if (typeNode != null && "flattened".equals(typeNode.asText())) { fieldObj.put("type", "flat_object"); + // flat_object does not support the ES flattened guard parameters + fieldObj.remove("ignore_above"); + fieldObj.remove("depth_limit"); LOG.debug( "Transformed field '{}' from 'flattened' to 'flat_object'", entry.getKey()); } + // OpenSearch's boolean type does not support ignore_malformed (Elasticsearch does) + if (typeNode != null && "boolean".equals(typeNode.asText())) { + fieldObj.remove("ignore_malformed"); + } // Recurse into nested properties JsonNode nestedProps = fieldObj.get("properties"); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SystemRepositoryReindexStatusTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SystemRepositoryReindexStatusTest.java index 17bd9830ca26..f9761bda5c54 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SystemRepositoryReindexStatusTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SystemRepositoryReindexStatusTest.java @@ -79,7 +79,8 @@ void reindexMessageCleanStateMentionsNoOrphansAndHealthyCluster() { SearchReindexStatus status = new SearchReindexStatus(List.of(), 0, List.of(), List.of(), true, true); String message = SystemRepository.buildReindexStatusMessage(status); - assertTrue(message.contains("All deployed indexes were built from the current code mappings.")); + assertTrue( + message.contains("All deployed indexes were built from the current index mappings.")); assertTrue(message.toLowerCase().contains("no orphan indexes")); assertTrue(message.toLowerCase().contains("cluster healthy")); } @@ -114,7 +115,7 @@ void driftComputeFailureIsNotReportedAsUpToDate() { String message = SystemRepository.buildReindexStatusMessage(status); assertTrue(message.toLowerCase().contains("could not determine reindex status")); assertFalse( - message.contains("All deployed indexes were built from the current code mappings.")); + message.contains("All deployed indexes were built from the current index mappings.")); } @Test diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/CustomPropertySearchFieldsTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/CustomPropertySearchFieldsTest.java new file mode 100644 index 000000000000..c6d90cfe534b --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/CustomPropertySearchFieldsTest.java @@ -0,0 +1,98 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.search; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import org.junit.jupiter.api.Test; +import org.openmetadata.schema.api.search.AssetTypeConfiguration; +import org.openmetadata.schema.api.search.FieldBoost; + +class CustomPropertySearchFieldsTest { + + private static AssetTypeConfiguration config(FieldBoost... fields) { + return new AssetTypeConfiguration().withSearchFields(List.of(fields)); + } + + private static FieldBoost field(String name, Double boost, FieldBoost.MatchType matchType) { + return new FieldBoost().withField(name).withBoost(boost).withMatchType(matchType); + } + + @Test + void resolvesExtensionFieldToNameBoostAndStandardMatch() { + List specs = + CustomPropertySearchFields.from( + config(field("extension.reviewer", 20.0, FieldBoost.MatchType.STANDARD))); + + assertEquals(1, specs.size()); + assertEquals("reviewer", specs.getFirst().propertyName()); + assertEquals(20.0f, specs.getFirst().boost()); + assertFalse(specs.getFirst().exact()); + } + + @Test + void marksExactMatchTypeAsExact() { + List specs = + CustomPropertySearchFields.from( + config(field("extension.code", 1.0, FieldBoost.MatchType.EXACT))); + + assertTrue(specs.getFirst().exact()); + } + + @Test + void ignoresNonExtensionFields() { + List specs = + CustomPropertySearchFields.from( + config( + field("name", 10.0, FieldBoost.MatchType.STANDARD), + field("description", 2.0, FieldBoost.MatchType.STANDARD))); + + assertTrue(specs.isEmpty()); + } + + @Test + void ignoresBareExtensionPrefixWithoutPropertyName() { + List specs = + CustomPropertySearchFields.from( + config(field("extension.", 1.0, FieldBoost.MatchType.STANDARD))); + + assertTrue(specs.isEmpty()); + } + + @Test + void keepsDottedPropertyNameAfterPrefix() { + List specs = + CustomPropertySearchFields.from( + config(field("extension.team.lead", 1.0, FieldBoost.MatchType.STANDARD))); + + assertEquals("team.lead", specs.getFirst().propertyName()); + } + + @Test + void defaultsBoostToOneWhenUnset() { + List specs = + CustomPropertySearchFields.from( + config(field("extension.reviewer", null, FieldBoost.MatchType.STANDARD))); + + assertEquals(1.0f, specs.getFirst().boost()); + } + + @Test + void returnsEmptyForNullConfigOrFields() { + assertTrue(CustomPropertySearchFields.from(null).isEmpty()); + assertTrue(CustomPropertySearchFields.from(new AssetTypeConfiguration()).isEmpty()); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/ImmenseTermReproTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/ImmenseTermReproTest.java new file mode 100644 index 000000000000..b50a1d081de6 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/ImmenseTermReproTest.java @@ -0,0 +1,165 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.search; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import java.io.InputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.openmetadata.service.search.opensearch.OsUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +/** + * Reproduces the production "immense term" reindex failures (SPG-DEV / crocs) against a real + * OpenSearch node and proves the fix. The custom-property value that triggered the incidents lands in + * two indexed fields: + * + *
    + *
  • {@code customPropertiesTyped.stringValue} — a nested {@code keyword}. Fixed by {@code + * ignore_above} (added by {@link SearchIndexSettings#harden}). + *
  • {@code extension} — formerly {@code flattened}/{@code flat_object}, which OpenSearch cannot + * guard with {@code ignore_above}. Fixed by making it {@code object, enabled:false} (stored, + * not indexed), mirroring the {@code columns.children} fix. + *
+ * + * The end-to-end test loads the real table mapping, hardens it, transforms it for OpenSearch and + * confirms the oversized document is accepted. + */ +class ImmenseTermReproTest { + + private static final Logger LOG = LoggerFactory.getLogger(ImmenseTermReproTest.class); + + private static final String OVERSIZED = "a".repeat(70000); // > 32766 byte Lucene term limit + private static final HttpClient HTTP = HttpClient.newHttpClient(); + + private static GenericContainer opensearch; + private static String baseUrl; + + @BeforeAll + static void startOpenSearch() { + assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required"); + opensearch = + new GenericContainer<>(DockerImageName.parse("opensearchproject/opensearch:2.13.0")) + .withEnv("discovery.type", "single-node") + .withEnv("DISABLE_SECURITY_PLUGIN", "true") + .withEnv("DISABLE_INSTALL_DEMO_CONFIG", "true") + .withEnv("OPENSEARCH_JAVA_OPTS", "-Xms1g -Xmx1g") + .withExposedPorts(9200) + .waitingFor(Wait.forHttp("/").forPort(9200).forStatusCode(200)) + .withStartupTimeout(Duration.ofMinutes(3)); + opensearch.start(); + baseUrl = "http://" + opensearch.getHost() + ":" + opensearch.getMappedPort(9200); + } + + @AfterAll + static void stopOpenSearch() { + if (opensearch != null) { + opensearch.stop(); + } + } + + @Test + void customPropertiesTyped_keyword_reproducesImmenseTerm_andHardeningFixesIt() throws Exception { + String rawMapping = + "{\"mappings\":{\"properties\":{\"customPropertiesTyped\":{\"type\":\"nested\"," + + "\"properties\":{\"stringValue\":{\"type\":\"keyword\"}}}}}}"; + String doc = "{\"customPropertiesTyped\":[{\"stringValue\":\"" + OVERSIZED + "\"}]}"; + + assertImmenseTerm(indexInto("cpt_raw", rawMapping, doc)); + + String hardened = OsUtils.enrichIndexMappingForOpenSearch(harden(rawMapping)); + assertEquals( + 201, indexInto("cpt_fixed", hardened, doc).statusCode(), "ignore_above must accept"); + } + + @Test + void extension_flatObject_reproducesImmenseTerm_andObjectEnabledFalseFixesIt() throws Exception { + String doc = "{\"extension\":{\"BodyType\":\"" + OVERSIZED + "\"}}"; + + String flatObject = + "{\"mappings\":{\"properties\":{\"extension\":{\"type\":\"flat_object\"}}}}"; + assertImmenseTerm(indexInto("ext_raw", flatObject, doc)); + + String objectDisabled = + "{\"mappings\":{\"properties\":{\"extension\":{\"type\":\"object\",\"enabled\":false}}}}"; + assertEquals( + 201, + indexInto("ext_fixed", objectDisabled, doc).statusCode(), + "object/enabled:false must accept"); + } + + @Test + void endToEnd_realTableMapping_hardened_acceptsOversizedDocument() throws Exception { + String raw; + try (InputStream is = + getClass() + .getClassLoader() + .getResourceAsStream("elasticsearch/en/table_index_mapping.json")) { + raw = new String(is.readAllBytes(), StandardCharsets.UTF_8); + } + String osMapping = OsUtils.enrichIndexMappingForOpenSearch(harden(raw)); + + String doc = + "{\"customPropertiesTyped\":[{\"name\":\"BodyType\",\"stringValue\":\"" + + OVERSIZED + + "\"}],\"extension\":{\"BodyType\":\"" + + OVERSIZED + + "\"}}"; + HttpResponse docResponse = indexInto("endtoend_table", osMapping, doc); + + LOG.info("END-TO-END oversized doc: [{}]", docResponse.statusCode()); + assertEquals( + 201, docResponse.statusCode(), "real hardened mapping must accept the oversized doc"); + } + + private String harden(String mapping) { + return SearchIndexSettings.harden(mapping, SearchFieldLimits.defaults()); + } + + private void assertImmenseTerm(HttpResponse response) { + assertEquals(400, response.statusCode(), "engine must reject the oversized document"); + assertTrue( + response.body().contains("immense term") || response.body().contains("max length 32766"), + "rejection must be the Lucene immense-term error, got: " + response.body()); + } + + private HttpResponse indexInto(String index, String mapping, String doc) + throws Exception { + assertEquals(200, put("/" + index, mapping).statusCode(), "index " + index + " should create"); + return put("/" + index + "/_doc/1?refresh=true", doc); + } + + private HttpResponse put(String path, String body) throws Exception { + HttpRequest request = + HttpRequest.newBuilder(URI.create(baseUrl + path)) + .header("Content-Type", "application/json") + .PUT(HttpRequest.BodyPublishers.ofString(body)) + .build(); + return HTTP.send(request, HttpResponse.BodyHandlers.ofString()); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/IndexMappingNestedFieldConsistencyTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/IndexMappingNestedFieldConsistencyTest.java index a4d0a8ac6080..357a656c85b0 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/search/IndexMappingNestedFieldConsistencyTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/IndexMappingNestedFieldConsistencyTest.java @@ -50,7 +50,7 @@ static void loadAllMappings() throws IOException { } @Test - void extensionFieldMustBeFlattenedInAllIndices() { + void extensionFieldMustBeDisabledObjectInAllIndices() { List violations = new ArrayList<>(); for (Map.Entry entry : allMappings.entrySet()) { String entity = entry.getKey(); @@ -62,9 +62,11 @@ void extensionFieldMustBeFlattenedInAllIndices() { } assertTrue( violations.isEmpty(), - "The 'extension' field must have \"type\": \"flattened\" in all index mappings. " - + "Using 'keyword' or 'object' will cause reindex failures when custom properties " - + "(entityExtension) contain object/map values. Violations: " + "The 'extension' field must be \"type\": \"object\" with \"enabled\": false in all index " + + "mappings. It stores arbitrary custom-property (entityExtension) values without " + + "indexing them, which avoids field explosion and the Lucene 32766-byte immense-term " + + "failure that flattened/flat_object leaves hit on OpenSearch. Custom-property search " + + "goes through customPropertiesTyped. Violations: " + violations); } @@ -131,9 +133,13 @@ private static void findExtensionTypeViolations( String path = currentPath.isEmpty() ? name : currentPath + "." + name; if (name.equals("extension")) { String type = fieldNode.path("type").asText(""); - if (!"flattened".equals(type)) { + boolean disabledObject = + "object".equals(type) && !fieldNode.path("enabled").asBoolean(true); + if (!disabledObject) { String detail = - type.isEmpty() ? "missing \"type\" (implicit object)" : "\"" + type + "\""; + type.isEmpty() + ? "missing \"type\" (implicit object)" + : "\"" + type + "\" enabled=" + fieldNode.path("enabled").asText("true"); violations.add(entity + " (" + path + "): " + detail); } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/IndexMappingVersionTrackerTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/IndexMappingVersionTrackerTest.java index 4be2f12247f2..f2315d2ea67d 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/search/IndexMappingVersionTrackerTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/IndexMappingVersionTrackerTest.java @@ -6,6 +6,7 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -31,10 +32,13 @@ import org.mockito.Captor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.openmetadata.schema.configuration.SearchIndexMappings; import org.openmetadata.search.IndexMapping; import org.openmetadata.search.IndexMappingLoader; +import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.IndexMappingVersionDAO; +import org.openmetadata.service.jdbi3.SystemRepository; @ExtendWith(MockitoExtension.class) class IndexMappingVersionTrackerTest { @@ -159,6 +163,48 @@ void getChangedMappingsReturnsEntitiesWithStaleHashes() throws IOException { } } + @Test + void storedMappingEditSurfacesAsDriftAgainstSeededDefault() throws IOException { + Map mappings = + buildMappingsFromPairs("table", "/elasticsearch/%s/table_index_mapping.json"); + try (var loaderMock = mockStatic(IndexMappingLoader.class); + var entityMock = mockStatic(Entity.class)) { + loaderMock.when(IndexMappingLoader::getInstance).thenReturn(indexMappingLoader); + when(indexMappingLoader.getIndexMapping()).thenReturn(mappings); + SystemRepository systemRepository = mock(SystemRepository.class); + entityMock.when(Entity::getSystemRepository).thenReturn(systemRepository); + + IndexMappingVersionTracker tracker = + new IndexMappingVersionTracker(collectionDAO, "1.2.3", "tester"); + + when(systemRepository.getSearchIndexMappings()).thenReturn(null); + tracker.updateMappingVersions(); + verify(indexMappingVersionDAO) + .upsertIndexMappingVersion( + eq("table"), hashCaptor.capture(), anyString(), eq("1.2.3"), anyLong(), eq("tester")); + String defaultHash = hashCaptor.getValue(); + when(indexMappingVersionDAO.getAllMappingVersions()) + .thenReturn( + List.of(new IndexMappingVersionDAO.IndexMappingVersion("table", defaultHash))); + + assertTrue( + tracker.getChangedMappings().isEmpty(), + "no drift when stored mapping matches the seeded default"); + + Map editedTable = + Map.of( + "mappings", Map.of("properties", Map.of("customField", Map.of("type", "keyword")))); + SearchIndexMappings editedBlob = + new SearchIndexMappings().withLanguages(Map.of("en", Map.of("table", editedTable))); + when(systemRepository.getSearchIndexMappings()).thenReturn(editedBlob); + + assertEquals( + List.of("table"), + tracker.getChangedMappings(), + "an admin edit to the stored mapping must surface as reindex-required drift"); + } + } + @Test void getChangedMappingsReturnsEmptyWhenHashesMatch() throws IOException { Map mappings = diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/SearchIndexMappingsSeederTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/SearchIndexMappingsSeederTest.java new file mode 100644 index 000000000000..9f54b1baefcf --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/SearchIndexMappingsSeederTest.java @@ -0,0 +1,77 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.search; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.openmetadata.schema.configuration.SearchIndexMappings; +import org.openmetadata.schema.settings.Settings; +import org.openmetadata.schema.settings.SettingsType; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.search.IndexMappingLoader; +import org.openmetadata.service.jdbi3.CollectionDAO; + +class SearchIndexMappingsSeederTest { + + @BeforeAll + static void init() throws Exception { + IndexMappingLoader.init(); + SearchFieldLimits.setActive(SearchFieldLimits.defaults()); + } + + @Test + void buildDefaultBlob_seedsEntitiesAndBakesInGuards() { + SearchIndexMappings blob = SearchIndexMappingsSeeder.buildDefaultBlob(List.of("en")); + + assertNotNull(blob.getLanguages()); + Map enMappings = blob.getLanguages().get("en"); + assertNotNull(enMappings, "en mappings present"); + assertTrue(enMappings.containsKey("table"), "table mapping seeded"); + + String tableJson = JsonUtils.pojoToJson(enMappings.get("table")); + assertTrue( + tableJson.contains("ignore_above"), + "keyword fields are hardened with ignore_above at seed"); + } + + @Test + void buildEntityMapping_returnsHardenedMapping_andNullForUnknownEntity() { + Map table = SearchIndexMappingsSeeder.buildEntityMapping("en", "table"); + + assertNotNull(table); + assertTrue(table.containsKey("mappings")); + assertNull(SearchIndexMappingsSeeder.buildEntityMapping("en", "no_such_entity")); + } + + @Test + void settingsRowMapper_roundTripsSearchIndexMappings() { + SearchIndexMappings blob = SearchIndexMappingsSeeder.buildDefaultBlob(List.of("en")); + String json = JsonUtils.pojoToJson(blob); + + Settings settings = + CollectionDAO.SettingsRowMapper.getSettings(SettingsType.SEARCH_INDEX_MAPPINGS, json); + + assertNotNull(settings); + SearchIndexMappings roundTripped = + JsonUtils.convertValue(settings.getConfigValue(), SearchIndexMappings.class); + assertNotNull(roundTripped.getLanguages().get("en")); + assertFalse(roundTripped.getLanguages().get("en").isEmpty()); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/SearchIndexSettingsTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/SearchIndexSettingsTest.java new file mode 100644 index 000000000000..1656a48fb403 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/SearchIndexSettingsTest.java @@ -0,0 +1,210 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.search; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration; +import org.openmetadata.schema.service.configuration.elasticsearch.SearchIndexingLimits; +import org.openmetadata.schema.utils.JsonUtils; + +class SearchIndexSettingsTest { + + private SearchFieldLimits limitsWith(int depth, int nestedObjects, int totalFields) { + SearchIndexingLimits limits = + new SearchIndexingLimits() + .withMappingDepthLimit(depth) + .withNestedObjectsLimit(nestedObjects) + .withTotalFieldsLimit(totalFields); + return SearchFieldLimits.from( + new ElasticSearchConfiguration().withSearchIndexingLimits(limits)); + } + + private JsonNode mappingLimits(String content) { + return JsonUtils.readTree(content).get("settings").get("index").get("mapping"); + } + + private JsonNode properties(String content) { + return JsonUtils.readTree(content).get("mappings").get("properties"); + } + + @Test + void injects_limits_and_preserves_existing_settings() { + String content = "{\"settings\":{\"index\":{\"max_ngram_diff\":17}},\"mappings\":{}}"; + + String result = SearchIndexSettings.harden(content, limitsWith(15, 5000, 2000)); + + JsonNode root = JsonUtils.readTree(result); + assertEquals(17, root.get("settings").get("index").get("max_ngram_diff").asInt()); + JsonNode mapping = mappingLimits(result); + assertEquals(15, mapping.get("depth").get("limit").asInt()); + assertEquals(5000, mapping.get("nested_objects").get("limit").asInt()); + assertEquals(2000, mapping.get("total_fields").get("limit").asInt()); + } + + @Test + void does_not_override_existing_limit() { + String content = + "{\"settings\":{\"index\":{\"mapping\":{\"depth\":{\"limit\":7}}}},\"mappings\":{}}"; + + String result = SearchIndexSettings.harden(content, limitsWith(15, 5000, 2000)); + + assertEquals(7, mappingLimits(result).get("depth").get("limit").asInt()); + } + + @Test + void adds_ignore_above_to_keyword_fields_and_multifields() { + String content = + "{\"mappings\":{\"properties\":{" + + "\"fullyQualifiedName\":{\"type\":\"keyword\"}," + + "\"name\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\"}}}" + + "}}}"; + + String result = SearchIndexSettings.harden(content, SearchFieldLimits.defaults()); + + JsonNode props = properties(result); + int expected = SearchFieldLimits.defaults().getSafeCharThreshold(); + assertEquals(expected, props.get("fullyQualifiedName").get("ignore_above").asInt()); + assertEquals( + expected, props.get("name").get("fields").get("keyword").get("ignore_above").asInt()); + } + + @Test + void does_not_override_existing_ignore_above() { + String content = + "{\"mappings\":{\"properties\":{\"tagFQN\":{\"type\":\"keyword\",\"ignore_above\":256}}}}"; + + String result = SearchIndexSettings.harden(content, SearchFieldLimits.defaults()); + + assertEquals(256, properties(result).get("tagFQN").get("ignore_above").asInt()); + } + + @Test + void adds_ignore_malformed_to_numeric_and_date_fields() { + String content = + "{\"mappings\":{\"properties\":{" + + "\"count\":{\"type\":\"integer\"}," + + "\"created\":{\"type\":\"date\"}," + + "\"description\":{\"type\":\"text\"}" + + "}}}"; + + String result = SearchIndexSettings.harden(content, SearchFieldLimits.defaults()); + + JsonNode props = properties(result); + assertTrue(props.get("count").get("ignore_malformed").asBoolean()); + assertTrue(props.get("created").get("ignore_malformed").asBoolean()); + assertFalse( + props.get("description").has("ignore_malformed"), "text fields must not be guarded"); + } + + @Test + void hardens_nested_object_properties() { + String content = + "{\"mappings\":{\"properties\":{\"tags\":{\"type\":\"nested\",\"properties\":{" + + "\"tagFQN\":{\"type\":\"keyword\"}}}}}}"; + + String result = SearchIndexSettings.harden(content, SearchFieldLimits.defaults()); + + JsonNode tagFqn = properties(result).get("tags").get("properties").get("tagFQN"); + assertEquals( + SearchFieldLimits.defaults().getSafeCharThreshold(), tagFqn.get("ignore_above").asInt()); + } + + @Test + void guards_flattened_fields_with_ignore_above_and_depth_limit() { + String content = "{\"mappings\":{\"properties\":{\"extension\":{\"type\":\"flattened\"}}}}"; + SearchFieldLimits limits = SearchFieldLimits.defaults(); + + JsonNode extension = properties(SearchIndexSettings.harden(content, limits)).get("extension"); + + assertEquals(limits.getSafeCharThreshold(), extension.get("ignore_above").asInt()); + assertEquals(limits.getDepthLimit(), extension.get("depth_limit").asInt()); + } + + @Test + void guards_column_level_extension() { + String content = + "{\"mappings\":{\"properties\":{\"columns\":{\"properties\":{" + + "\"extension\":{\"type\":\"flattened\"}}}}}}"; + + String result = SearchIndexSettings.harden(content, SearchFieldLimits.defaults()); + + JsonNode columnExtension = properties(result).get("columns").get("properties").get("extension"); + assertTrue(columnExtension.has("ignore_above"), "nested column extension must be guarded"); + } + + @Test + void ignore_above_is_never_zero_for_tiny_keyword_max_bytes() { + SearchFieldLimits limits = + SearchFieldLimits.from( + new ElasticSearchConfiguration() + .withSearchIndexingLimits(new SearchIndexingLimits().withKeywordMaxBytes(1))); + String content = "{\"mappings\":{\"properties\":{\"fqn\":{\"type\":\"keyword\"}}}}"; + + int ignoreAbove = + properties(SearchIndexSettings.harden(content, limits)) + .get("fqn") + .get("ignore_above") + .asInt(); + + assertTrue(ignoreAbove >= 1, "ignore_above must be at least 1"); + } + + @Test + void returns_null_content_unchanged() { + assertNull(SearchIndexSettings.harden(null, SearchFieldLimits.defaults())); + } + + /** + * Covers every Elasticsearch/OpenSearch field type used across the index mappings: keyword and + * flattened get ignore_above; numeric/date/boolean types get ignore_malformed; container and + * special types (text, nested, object, completion) get neither directly. + */ + @ParameterizedTest(name = "{0}") + @CsvSource({ + "keyword, true, false", + "text, false, false", + "long, false, true", + "integer, false, true", + "short, false, true", + "byte, false, true", + "double, false, true", + "float, false, true", + "half_float, false, true", + "scaled_float, false, true", + "date, false, true", + "boolean, false, true", + "nested, false, false", + "object, false, false", + "flattened, true, false", + "completion, false, false" + }) + void hardens_each_field_type_correctly( + String type, boolean expectIgnoreAbove, boolean expectIgnoreMalformed) { + String content = "{\"mappings\":{\"properties\":{\"f\":{\"type\":\"" + type + "\"}}}}"; + + String result = SearchIndexSettings.harden(content, SearchFieldLimits.defaults()); + + JsonNode field = properties(result).get("f"); + assertEquals(expectIgnoreAbove, field.has("ignore_above"), "ignore_above for type " + type); + assertEquals( + expectIgnoreMalformed, field.has("ignore_malformed"), "ignore_malformed for type " + type); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/ColumnIndexLimitTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/ColumnIndexLimitTest.java new file mode 100644 index 000000000000..9973dd016140 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/ColumnIndexLimitTest.java @@ -0,0 +1,164 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.search.indexes; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration; +import org.openmetadata.schema.service.configuration.elasticsearch.SearchIndexingLimits; +import org.openmetadata.schema.type.Column; +import org.openmetadata.schema.type.TagLabel; +import org.openmetadata.service.Entity; +import org.openmetadata.service.search.SearchFieldLimits; +import org.openmetadata.service.search.SearchRepository; +import org.openmetadata.service.search.models.FlattenColumn; + +class ColumnIndexLimitTest { + + private static SearchRepository previousRepository; + + @BeforeAll + static void initSearchRepository() { + previousRepository = Entity.getSearchRepository(); + Entity.setSearchRepository(mock(SearchRepository.class)); + } + + @AfterAll + static void restoreSearchRepository() { + Entity.setSearchRepository(previousRepository); + } + + private static final class TestColumnIndex implements ColumnIndex { + @Override + public Object getEntity() { + return null; + } + + @Override + public String getEntityTypeName() { + return "table"; + } + + @Override + public Map buildSearchIndexDocInternal(Map esDoc) { + return esDoc; + } + } + + @AfterEach + void resetLimits() { + SearchFieldLimits.setActive(null); + } + + private void activateLimits(int depth, int maxColumns) { + SearchIndexingLimits limits = + new SearchIndexingLimits().withMappingDepthLimit(depth).withMaxColumns(maxColumns); + SearchFieldLimits.setActive( + SearchFieldLimits.from(new ElasticSearchConfiguration().withSearchIndexingLimits(limits))); + } + + private Column nestedChain(int depth) { + Column current = new Column().withName("c" + depth); + Column root = current; + for (int level = depth - 1; level >= 1; level--) { + Column parent = new Column().withName("c" + level).withChildren(List.of(current)); + current = parent; + root = parent; + } + return root; + } + + @Test + void parseColumns_stops_at_depth_limit() { + activateLimits(3, 10000); + List flattened = new ArrayList<>(); + + new TestColumnIndex().parseColumns(List.of(nestedChain(5)), flattened, null); + + assertEquals(3, flattened.size(), "recursion must stop at depth limit"); + } + + @Test + void parseColumns_does_not_leak_tags_between_siblings() { + activateLimits(20, 10000); + Column tagged = + new Column().withName("a").withTags(List.of(new TagLabel().withTagFQN("PII.Sensitive"))); + Column untagged = new Column().withName("b"); + List flattened = new ArrayList<>(); + + new TestColumnIndex().parseColumns(List.of(tagged, untagged), flattened, null); + + assertEquals("a", flattened.get(0).getName()); + assertEquals(1, flattened.get(0).getTags().size(), "tagged column keeps its tag"); + assertTrue( + flattened.get(1).getTags() == null || flattened.get(1).getTags().isEmpty(), + "untagged sibling must not inherit the previous column's tags"); + } + + @Test + void parseColumns_builds_fully_qualified_names() { + activateLimits(20, 10000); + List flattened = new ArrayList<>(); + + new TestColumnIndex().parseColumns(List.of(nestedChain(3)), flattened, null); + + assertEquals("c1", flattened.get(0).getName()); + assertEquals("c1.c2", flattened.get(1).getName()); + assertEquals("c1.c2.c3", flattened.get(2).getName(), "deep path must keep full prefix"); + } + + @Test + void parseColumns_stops_at_max_columns() { + activateLimits(20, 4); + List siblings = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + siblings.add(new Column().withName("col" + i)); + } + List flattened = new ArrayList<>(); + + new TestColumnIndex().parseColumns(siblings, flattened, null); + + assertEquals(4, flattened.size(), "must stop once max columns reached"); + } + + @Test + void flattenColumns_respects_depth_limit() { + activateLimits(3, 10000); + + List result = ColumnSearchIndex.flattenColumns(List.of(nestedChain(5))); + + assertEquals(3, result.size(), "static flatten must stop at depth limit"); + } + + @Test + void flattenColumns_respects_max_columns() { + activateLimits(20, 4); + List siblings = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + siblings.add(new Column().withName("col" + i)); + } + + List result = ColumnSearchIndex.flattenColumns(siblings); + + assertEquals(4, result.size(), "static flatten must stop at max columns"); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/IngestionPipelineIndexTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/IngestionPipelineIndexTest.java new file mode 100644 index 000000000000..825bba109b6b --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/IngestionPipelineIndexTest.java @@ -0,0 +1,74 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.search.indexes; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.mockito.Mockito.mock; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; +import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineStatus; +import org.openmetadata.schema.metadataIngestion.SourceConfig; +import org.openmetadata.service.Entity; +import org.openmetadata.service.search.SearchRepository; + +class IngestionPipelineIndexTest { + + private static SearchRepository previousRepository; + + @BeforeAll + static void initSearchRepository() { + previousRepository = Entity.getSearchRepository(); + Entity.setSearchRepository(mock(SearchRepository.class)); + } + + @AfterAll + static void restoreSearchRepository() { + Entity.setSearchRepository(previousRepository); + } + + @Test + void buildDoc_stripsFreeFormFieldsFromPipelineStatuses_keepsOtherFields() { + PipelineStatus status = + new PipelineStatus() + .withRunId("run-1") + .withConfig( + Map.of("appConfig", Map.of("actions", Map.of("customProperties", "jointure")))) + .withMetadata(Map.of("arbitraryKey", Map.of("nested", "value"))); + IngestionPipeline pipeline = + new IngestionPipeline() + .withName("p1") + .withPipelineStatuses(List.of(status)) + .withSourceConfig(new SourceConfig().withConfig(Map.of())); + + Map doc = + new IngestionPipelineIndex(pipeline).buildSearchIndexDocInternal(new HashMap<>()); + + Object pipelineStatuses = doc.get("pipelineStatuses"); + assertInstanceOf(List.class, pipelineStatuses); + List statuses = (List) pipelineStatuses; + assertEquals(1, statuses.size()); + Map statusMap = (Map) statuses.getFirst(); + assertFalse(statusMap.containsKey("config"), "free-form config must be stripped from the doc"); + assertFalse( + statusMap.containsKey("metadata"), "free-form metadata must be stripped from the doc"); + assertEquals("run-1", statusMap.get("runId"), "searchable status fields must be preserved"); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/SchemaFieldFlattenerTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/SchemaFieldFlattenerTest.java new file mode 100644 index 000000000000..19bb0903db96 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/SchemaFieldFlattenerTest.java @@ -0,0 +1,87 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.search.indexes; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration; +import org.openmetadata.schema.service.configuration.elasticsearch.SearchIndexingLimits; +import org.openmetadata.schema.type.Field; +import org.openmetadata.service.search.SearchFieldLimits; +import org.openmetadata.service.search.models.FlattenSchemaField; + +class SchemaFieldFlattenerTest { + + @AfterEach + void resetLimits() { + SearchFieldLimits.setActive(null); + } + + private void activateLimits(int depth, int maxFields) { + SearchIndexingLimits limits = + new SearchIndexingLimits().withMappingDepthLimit(depth).withMaxColumns(maxFields); + SearchFieldLimits.setActive( + SearchFieldLimits.from(new ElasticSearchConfiguration().withSearchIndexingLimits(limits))); + } + + private Field nestedChain(int depth) { + Field current = new Field().withName("f" + depth); + Field root = current; + for (int level = depth - 1; level >= 1; level--) { + Field parent = new Field().withName("f" + level).withChildren(List.of(current)); + current = parent; + root = parent; + } + return root; + } + + @Test + void parseSchemaFields_stops_at_depth_limit() { + activateLimits(3, 10000); + List flattened = new ArrayList<>(); + + SchemaFieldFlattener.parseSchemaFields(List.of(nestedChain(5)), flattened, null); + + assertEquals(3, flattened.size(), "recursion must stop at depth limit"); + } + + @Test + void parseSchemaFields_builds_fully_qualified_names() { + activateLimits(20, 10000); + List flattened = new ArrayList<>(); + + SchemaFieldFlattener.parseSchemaFields(List.of(nestedChain(3)), flattened, null); + + assertEquals("f1", flattened.get(0).getName()); + assertEquals("f1.f2", flattened.get(1).getName()); + assertEquals("f1.f2.f3", flattened.get(2).getName(), "deep path must keep full prefix"); + } + + @Test + void parseSchemaFields_stops_at_max_fields() { + activateLimits(20, 4); + List siblings = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + siblings.add(new Field().withName("field" + i)); + } + List flattened = new ArrayList<>(); + + SchemaFieldFlattener.parseSchemaFields(siblings, flattened, null); + + assertEquals(4, flattened.size(), "must stop once max fields reached"); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/opensearch/OsUtilsTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/opensearch/OsUtilsTest.java index e0a25d633171..e2f3027e4caf 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/search/opensearch/OsUtilsTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/opensearch/OsUtilsTest.java @@ -842,4 +842,19 @@ void testAddKnnVectorSettingsAddsEmbeddingMetadataFromClientDimension() { .asInt()); } } + + @Test + void transformsFlattenedToFlatObjectAndStripsEsOnlyParams() { + JsonNode mappings = + org.openmetadata.schema.utils.JsonUtils.readTree( + "{\"properties\":{\"extension\":{\"type\":\"flattened\",\"ignore_above\":8191," + + "\"depth_limit\":20}}}"); + + JsonNode result = OsUtils.transformFieldTypesForOpenSearch(mappings); + + JsonNode extension = result.path("properties").path("extension"); + assertEquals("flat_object", extension.path("type").asText()); + assertFalse(extension.has("ignore_above"), "flat_object must not carry ignore_above"); + assertFalse(extension.has("depth_limit"), "flat_object must not carry depth_limit"); + } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/ai_agent_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/ai_agent_index_mapping.json index bbc2b9090ef6..137bb2b85037 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/ai_agent_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/ai_agent_index_mapping.json @@ -260,7 +260,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "mlFeatures": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/ai_governance_policy_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/ai_governance_policy_index_mapping.json index c9a2b9fc72ac..dbf1b45f408f 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/ai_governance_policy_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/ai_governance_policy_index_mapping.json @@ -260,7 +260,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "mlFeatures": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/api_collection_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/api_collection_index_mapping.json index 72a53c884648..491ab397d627 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/api_collection_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/api_collection_index_mapping.json @@ -331,7 +331,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "owners": { "type": "nested", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/api_endpoint_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/api_endpoint_index_mapping.json index 650793f20a5c..8bc153e6d726 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/api_endpoint_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/api_endpoint_index_mapping.json @@ -253,7 +253,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "dataProducts": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/column_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/column_index_mapping.json index 51b337389a2a..c5fe06a805a9 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/column_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/column_index_mapping.json @@ -584,7 +584,8 @@ "normalizer": "lowercase_normalizer" }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "customPropertiesTyped": { "type": "nested", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/container_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/container_index_mapping.json index 2e69470e5eb1..964fdd08fefa 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/container_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/container_index_mapping.json @@ -399,7 +399,8 @@ "type": "integer" }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "customPropertiesTyped": { "type": "nested", @@ -553,7 +554,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "children": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/context_file_search_index.json b/openmetadata-spec/src/main/resources/elasticsearch/en/context_file_search_index.json index 03b6d0d7c715..e082f0ebce7d 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/context_file_search_index.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/context_file_search_index.json @@ -339,7 +339,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "path": { "type": "text", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/context_memory_search_index.json b/openmetadata-spec/src/main/resources/elasticsearch/en/context_memory_search_index.json index 6f739bebee07..a0ce7092695d 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/context_memory_search_index.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/context_memory_search_index.json @@ -636,7 +636,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "fingerprint": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json index 82068e70ec64..6c72e9244eeb 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json @@ -568,7 +568,8 @@ "type": "integer" }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "customPropertiesTyped": { "type": "nested", @@ -687,7 +688,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "totalVotes": { "type": "long", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/dashboard_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/dashboard_index_mapping.json index 29a367ff877f..e78ff13b3dfc 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/dashboard_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/dashboard_index_mapping.json @@ -441,7 +441,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "dataProducts": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/data_products_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/data_products_index_mapping.json index d19f3ae960c9..be078cae4e45 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/data_products_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/data_products_index_mapping.json @@ -528,7 +528,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "certification": { "type": "object", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/database_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/database_index_mapping.json index 403cc26438c6..2d18a0ef8c98 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/database_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/database_index_mapping.json @@ -331,7 +331,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "owners": { "type": "nested", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/database_schema_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/database_schema_index_mapping.json index 6e51ed087e7b..9adb62e43064 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/database_schema_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/database_schema_index_mapping.json @@ -268,7 +268,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "owners": { "type": "nested", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/directory_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/directory_index_mapping.json index 627417961b83..b327f21f581f 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/directory_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/directory_index_mapping.json @@ -266,7 +266,8 @@ "type": "text" }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "parent": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/domain_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/domain_index_mapping.json index afa9060a0b2e..103c8700d8f4 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/domain_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/domain_index_mapping.json @@ -370,7 +370,8 @@ "dynamic": false }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "certification": { "type": "object", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/file_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/file_index_mapping.json index 4e46936b5d58..e525bf5bd0b6 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/file_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/file_index_mapping.json @@ -317,7 +317,8 @@ "type": "keyword" }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "path": { "type": "text", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/folder_search_index.json b/openmetadata-spec/src/main/resources/elasticsearch/en/folder_search_index.json index b9465d78194c..ad559680d188 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/folder_search_index.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/folder_search_index.json @@ -266,7 +266,8 @@ "type": "text" }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "parent": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/llm_model_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/llm_model_index_mapping.json index c9a2b9fc72ac..dbf1b45f408f 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/llm_model_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/llm_model_index_mapping.json @@ -260,7 +260,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "mlFeatures": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/mlmodel_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/mlmodel_index_mapping.json index 5b9d4f1f1ed8..c73d19943ade 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/mlmodel_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/mlmodel_index_mapping.json @@ -381,7 +381,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "mlFeatures": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/pipeline_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/pipeline_index_mapping.json index 57af7a9fb9f5..65e784c32e35 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/pipeline_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/pipeline_index_mapping.json @@ -424,7 +424,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "owners": { "type": "nested", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/prompt_template_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/prompt_template_index_mapping.json index c9a2b9fc72ac..dbf1b45f408f 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/prompt_template_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/prompt_template_index_mapping.json @@ -260,7 +260,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "mlFeatures": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/search_entity_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/search_entity_index_mapping.json index 3f148fe8aa5b..f1fd86f36d26 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/search_entity_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/search_entity_index_mapping.json @@ -577,7 +577,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "dataProducts": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/spreadsheet_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/spreadsheet_index_mapping.json index be6d96f5fc0d..cbcae05695c4 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/spreadsheet_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/spreadsheet_index_mapping.json @@ -266,7 +266,8 @@ "type": "text" }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "directory": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/stored_procedure_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/stored_procedure_index_mapping.json index 1f6bd5ab7e37..144e0bd178b8 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/stored_procedure_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/stored_procedure_index_mapping.json @@ -526,7 +526,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "sourceUrl": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/table_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/table_index_mapping.json index 6aff007662e0..075d117a5de2 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/table_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/table_index_mapping.json @@ -241,7 +241,8 @@ "type": "integer" }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "customPropertiesTyped": { "type": "nested", @@ -553,7 +554,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "dataProducts": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/topic_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/topic_index_mapping.json index c76477644fa5..9ebadeaabe2a 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/topic_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/topic_index_mapping.json @@ -209,7 +209,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "dataProducts": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/worksheet_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/worksheet_index_mapping.json index c08464a0d1fc..e4ade7557c7e 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/worksheet_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/worksheet_index_mapping.json @@ -266,7 +266,8 @@ "type": "text" }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "spreadsheet": { "properties": { @@ -412,7 +413,8 @@ "type": "integer" }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "customPropertiesTyped": { "type": "nested", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/ai_agent_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/ai_agent_index_mapping.json index 82fdf3c84e22..f1e36f5b606e 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/ai_agent_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/ai_agent_index_mapping.json @@ -265,7 +265,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "mlFeatures": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/ai_governance_policy_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/ai_governance_policy_index_mapping.json index e9dff9b2b828..86b7f95bee21 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/ai_governance_policy_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/ai_governance_policy_index_mapping.json @@ -265,7 +265,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "mlFeatures": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/api_collection_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/api_collection_index_mapping.json index 189579607d66..d2f0bc19934f 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/api_collection_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/api_collection_index_mapping.json @@ -255,7 +255,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "entityType": { "type": "keyword", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/api_endpoint_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/api_endpoint_index_mapping.json index 941a41074545..590a65dad020 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/api_endpoint_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/api_endpoint_index_mapping.json @@ -256,7 +256,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "dataProducts": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/column_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/column_index_mapping.json index ff1354c69910..775f68eda855 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/column_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/column_index_mapping.json @@ -589,7 +589,8 @@ "normalizer": "lowercase_normalizer" }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "customPropertiesTyped": { "type": "nested", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/container_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/container_index_mapping.json index 7754affd658a..9b57b1537ef2 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/container_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/container_index_mapping.json @@ -428,7 +428,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "dataModel": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/context_file_search_index.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/context_file_search_index.json index 03b6d0d7c715..e082f0ebce7d 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/context_file_search_index.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/context_file_search_index.json @@ -339,7 +339,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "path": { "type": "text", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/context_memory_search_index.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/context_memory_search_index.json index 6f739bebee07..a0ce7092695d 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/context_memory_search_index.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/context_memory_search_index.json @@ -636,7 +636,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "fingerprint": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json index ec3dee2c59b5..1b0de9e03efb 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json @@ -337,7 +337,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "dataProducts": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/data_products_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/data_products_index_mapping.json index 2065e5750605..f99ef35d91f5 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/data_products_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/data_products_index_mapping.json @@ -532,7 +532,8 @@ "normalizer": "lowercase_normalizer" }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "totalVotes": { "type": "long", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/database_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/database_index_mapping.json index 05ac8de4ca21..951486360ecb 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/database_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/database_index_mapping.json @@ -257,7 +257,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "entityType": { "type": "keyword", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/database_schema_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/database_schema_index_mapping.json index 01fe29b83e96..f3ffd9c0b77d 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/database_schema_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/database_schema_index_mapping.json @@ -272,7 +272,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "owners": { "type": "nested", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/domain_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/domain_index_mapping.json index 11f630ca222d..8c0f90702b82 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/domain_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/domain_index_mapping.json @@ -316,7 +316,8 @@ "normalizer": "lowercase_normalizer" }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "certification": { "type": "object", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/file_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/file_index_mapping.json index 367334c26146..69d2189081ad 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/file_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/file_index_mapping.json @@ -171,7 +171,8 @@ "type": "keyword" }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "path": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/llm_model_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/llm_model_index_mapping.json index e9dff9b2b828..86b7f95bee21 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/llm_model_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/llm_model_index_mapping.json @@ -265,7 +265,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "mlFeatures": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json index 1b45c88ab340..aa636dc483b2 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json @@ -384,7 +384,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "mlFeatures": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/prompt_template_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/prompt_template_index_mapping.json index e9dff9b2b828..86b7f95bee21 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/prompt_template_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/prompt_template_index_mapping.json @@ -265,7 +265,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "mlFeatures": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json index 25c7be3c84e3..f29dc3941827 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json @@ -456,7 +456,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "lifeCycle": { "type": "object", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/stored_procedure_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/stored_procedure_index_mapping.json index 4dd96dbf022b..c020bfc8640b 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/stored_procedure_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/stored_procedure_index_mapping.json @@ -618,7 +618,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "sourceUrl": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/table_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/table_index_mapping.json index e8f223624f2b..c070f10b0869 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/table_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/table_index_mapping.json @@ -231,7 +231,8 @@ "type": "integer" }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "customPropertiesTyped": { "type": "nested", @@ -543,7 +544,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "dataProducts": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/topic_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/topic_index_mapping.json index 3037a0ca63cc..4505b1b3f475 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/topic_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/topic_index_mapping.json @@ -529,7 +529,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "dataProducts": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/ai_agent_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/ai_agent_index_mapping.json index 19703c425022..3ac425742a04 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/ai_agent_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/ai_agent_index_mapping.json @@ -279,7 +279,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "mlFeatures": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/ai_governance_policy_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/ai_governance_policy_index_mapping.json index a0501dede4d4..03b72f249a98 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/ai_governance_policy_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/ai_governance_policy_index_mapping.json @@ -279,7 +279,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "mlFeatures": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/api_collection_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/api_collection_index_mapping.json index ac107cd2eea1..6ad91dacd3ea 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/api_collection_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/api_collection_index_mapping.json @@ -350,7 +350,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "owners": { "type": "nested", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/api_endpoint_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/api_endpoint_index_mapping.json index 44b29502567d..dacc029960cf 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/api_endpoint_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/api_endpoint_index_mapping.json @@ -272,7 +272,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "dataProducts": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/column_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/column_index_mapping.json index e9c81bc47c31..d31dcc344bb3 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/column_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/column_index_mapping.json @@ -603,7 +603,8 @@ "normalizer": "lowercase_normalizer" }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "customPropertiesTyped": { "type": "nested", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/container_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/container_index_mapping.json index 5bf0b06f403a..283388449743 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/container_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/container_index_mapping.json @@ -518,7 +518,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "children": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/context_file_search_index.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/context_file_search_index.json index 03b6d0d7c715..e082f0ebce7d 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/context_file_search_index.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/context_file_search_index.json @@ -339,7 +339,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "path": { "type": "text", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/context_memory_search_index.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/context_memory_search_index.json index 6f739bebee07..a0ce7092695d 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/context_memory_search_index.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/context_memory_search_index.json @@ -636,7 +636,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "fingerprint": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_data_model_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_data_model_index_mapping.json index 3a6287850bd9..af0c108df73c 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_data_model_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_data_model_index_mapping.json @@ -661,7 +661,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "totalVotes": { "type": "long", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_index_mapping.json index 3e6fe865de60..a535e3348a54 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_index_mapping.json @@ -460,7 +460,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "dataProducts": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/data_products_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/data_products_index_mapping.json index 28f68480c232..5dd98ede6c84 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/data_products_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/data_products_index_mapping.json @@ -541,7 +541,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "totalVotes": { "type": "long", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/database_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/database_index_mapping.json index ced4c8d14198..6e72fb739bec 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/database_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/database_index_mapping.json @@ -350,7 +350,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "owners": { "type": "nested", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/database_schema_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/database_schema_index_mapping.json index 9f93082a5e3e..5fa2d9deb11a 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/database_schema_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/database_schema_index_mapping.json @@ -287,7 +287,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "owners": { "type": "nested", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/domain_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/domain_index_mapping.json index c7947056d53c..691b009ede29 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/domain_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/domain_index_mapping.json @@ -389,7 +389,8 @@ "dynamic": false }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "certification": { "type": "object", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/file_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/file_index_mapping.json index 6f1e6550affe..40c83c7c225e 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/file_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/file_index_mapping.json @@ -264,7 +264,8 @@ "type": "keyword" }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "path": { "type": "text", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/llm_model_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/llm_model_index_mapping.json index a0501dede4d4..03b72f249a98 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/llm_model_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/llm_model_index_mapping.json @@ -279,7 +279,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "mlFeatures": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/mlmodel_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/mlmodel_index_mapping.json index 3ee8cb298151..c1e7a779c1e6 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/mlmodel_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/mlmodel_index_mapping.json @@ -400,7 +400,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "mlFeatures": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/pipeline_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/pipeline_index_mapping.json index 00b3eb8b36d2..5620011b7017 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/pipeline_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/pipeline_index_mapping.json @@ -443,7 +443,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "owners": { "type": "nested", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/prompt_template_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/prompt_template_index_mapping.json index a0501dede4d4..03b72f249a98 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/prompt_template_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/prompt_template_index_mapping.json @@ -279,7 +279,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "mlFeatures": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/search_entity_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/search_entity_index_mapping.json index 80a5f1a66281..4e323074eeaf 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/search_entity_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/search_entity_index_mapping.json @@ -596,7 +596,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "dataProducts": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/stored_procedure_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/stored_procedure_index_mapping.json index 9b0fc4c04ae9..1c7a9f1e1bb8 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/stored_procedure_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/stored_procedure_index_mapping.json @@ -545,7 +545,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "sourceUrl": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/table_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/table_index_mapping.json index 32775c2665de..2fde2f641b16 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/table_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/table_index_mapping.json @@ -251,7 +251,8 @@ "type": "integer" }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "customPropertiesTyped": { "type": "nested", @@ -563,7 +564,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "dataProducts": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/topic_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/topic_index_mapping.json index fde84081209d..4a2c76d1d62c 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/topic_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/topic_index_mapping.json @@ -228,7 +228,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "dataProducts": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/ai_agent_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/ai_agent_index_mapping.json index f9e78c0f364b..95475e280681 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/ai_agent_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/ai_agent_index_mapping.json @@ -255,7 +255,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "mlFeatures": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/ai_governance_policy_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/ai_governance_policy_index_mapping.json index bedc59536aba..b048b81a8b1b 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/ai_governance_policy_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/ai_governance_policy_index_mapping.json @@ -255,7 +255,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "mlFeatures": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/api_collection_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/api_collection_index_mapping.json index d6ba42dc7d92..a79fbebd75aa 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/api_collection_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/api_collection_index_mapping.json @@ -327,7 +327,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "owners": { "type": "nested", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/api_endpoint_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/api_endpoint_index_mapping.json index 64e15e994587..08b770c4f65f 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/api_endpoint_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/api_endpoint_index_mapping.json @@ -248,7 +248,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "dataProducts": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/column_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/column_index_mapping.json index 9dcf5d175991..2c697d606115 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/column_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/column_index_mapping.json @@ -581,7 +581,8 @@ "normalizer": "lowercase_normalizer" }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "customPropertiesTyped": { "type": "nested", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/container_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/container_index_mapping.json index d5fdd7615d42..038f71ca6030 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/container_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/container_index_mapping.json @@ -490,7 +490,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "children": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/context_file_search_index.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/context_file_search_index.json index 03b6d0d7c715..e082f0ebce7d 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/context_file_search_index.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/context_file_search_index.json @@ -339,7 +339,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "path": { "type": "text", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/context_memory_search_index.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/context_memory_search_index.json index 6f739bebee07..a0ce7092695d 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/context_memory_search_index.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/context_memory_search_index.json @@ -636,7 +636,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "fingerprint": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json index 527d39c9da66..73d75154d801 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json @@ -383,7 +383,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "dataProducts": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/data_products_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/data_products_index_mapping.json index 9e55b8c14279..e6ac5acbf1e0 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/data_products_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/data_products_index_mapping.json @@ -465,7 +465,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "tags": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/database_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/database_index_mapping.json index da40e43df134..82405a87089c 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/database_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/database_index_mapping.json @@ -328,7 +328,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "owners": { "type": "nested", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/database_schema_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/database_schema_index_mapping.json index f038e6e8c5ec..6e3f45fc026a 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/database_schema_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/database_schema_index_mapping.json @@ -265,7 +265,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "owners": { "type": "nested", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/domain_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/domain_index_mapping.json index 934264b9bd30..d27409d4cb96 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/domain_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/domain_index_mapping.json @@ -309,7 +309,8 @@ "normalizer": "lowercase_normalizer" }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "certification": { "type": "object", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/file_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/file_index_mapping.json index dc5de074685f..bbe86211678f 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/file_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/file_index_mapping.json @@ -144,7 +144,8 @@ "type": "keyword" }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "path": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/llm_model_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/llm_model_index_mapping.json index bedc59536aba..b048b81a8b1b 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/llm_model_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/llm_model_index_mapping.json @@ -255,7 +255,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "mlFeatures": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/mlmodel_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/mlmodel_index_mapping.json index a8cde779d689..c08f6d86cdb0 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/mlmodel_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/mlmodel_index_mapping.json @@ -375,7 +375,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "mlFeatures": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/prompt_template_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/prompt_template_index_mapping.json index bedc59536aba..b048b81a8b1b 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/prompt_template_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/prompt_template_index_mapping.json @@ -255,7 +255,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "mlFeatures": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json index c28e9a27dab6..944b8add57c1 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json @@ -574,7 +574,8 @@ "dynamic": false }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "dataProducts": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/stored_procedure_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/stored_procedure_index_mapping.json index 9148614b2f94..c9176ba52e8d 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/stored_procedure_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/stored_procedure_index_mapping.json @@ -615,7 +615,8 @@ "dynamic": false }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "sourceUrl": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/table_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/table_index_mapping.json index 25d8361357d9..c6b3c7af387e 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/table_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/table_index_mapping.json @@ -408,7 +408,8 @@ "type": "integer" }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "customPropertiesTyped": { "type": "nested", @@ -724,7 +725,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "dataProducts": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/topic_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/topic_index_mapping.json index 96914fa48d31..a60821cda583 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/topic_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/topic_index_mapping.json @@ -205,7 +205,8 @@ } }, "extension": { - "type": "flattened" + "type": "object", + "enabled": false }, "dataProducts": { "properties": { diff --git a/openmetadata-spec/src/main/resources/json/schema/configuration/elasticSearchConfiguration.json b/openmetadata-spec/src/main/resources/json/schema/configuration/elasticSearchConfiguration.json index de91a5ae1b4c..35f16033e34d 100644 --- a/openmetadata-spec/src/main/resources/json/schema/configuration/elasticSearchConfiguration.json +++ b/openmetadata-spec/src/main/resources/json/schema/configuration/elasticSearchConfiguration.json @@ -103,6 +103,49 @@ "description": "Index factory name", "type": "string" }, + "searchIndexingLimits": { + "description": "Limits applied while building search documents so that field values can never be rejected by Elasticsearch/OpenSearch. Values default to the documented engine defaults; override to tune without changing infrastructure settings.", + "type": "object", + "javaType": "org.openmetadata.schema.service.configuration.elasticsearch.SearchIndexingLimits", + "properties": { + "enableMappingHardening": { + "description": "Enable injecting ignore_above / ignore_malformed and index.mapping.*.limit guardrails into index mappings at creation time so documents cannot be rejected. When false, mappings are created as-is.", + "type": "boolean", + "default": true + }, + "keywordMaxBytes": { + "description": "Maximum UTF-8 byte length of a single keyword term. ignore_above is set to a byte-safe character count derived from this (value/4). The hard Lucene limit is 32766 bytes.", + "type": "integer", + "minimum": 4, + "default": 32766 + }, + "mappingDepthLimit": { + "description": "Maximum object/column nesting depth. Mirrors index.mapping.depth.limit.", + "type": "integer", + "minimum": 1, + "default": 20 + }, + "nestedObjectsLimit": { + "description": "Maximum number of nested-type objects allowed in a single document before Elasticsearch/OpenSearch rejects it (the engine rejects rather than truncates). Mirrors index.mapping.nested_objects.limit.", + "type": "integer", + "minimum": 1, + "default": 10000 + }, + "totalFieldsLimit": { + "description": "Maximum total fields per index. Mirrors index.mapping.total_fields.limit.", + "type": "integer", + "minimum": 1, + "default": 1000 + }, + "maxColumns": { + "description": "Maximum number of flattened columns or schema fields indexed for a single data asset. Items beyond this are dropped from the search document.", + "type": "integer", + "minimum": 1, + "default": 10000 + } + }, + "additionalProperties": false + }, "aws": { "description": "AWS IAM authentication configuration for OpenSearch. IAM auth must be explicitly enabled. When enabled, uses standard AWS environment variables or configured credentials.", "type": "object", diff --git a/openmetadata-spec/src/main/resources/json/schema/configuration/searchIndexMappings.json b/openmetadata-spec/src/main/resources/json/schema/configuration/searchIndexMappings.json new file mode 100644 index 000000000000..44a3de7e023a --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/configuration/searchIndexMappings.json @@ -0,0 +1,17 @@ +{ + "$id": "https://open-metadata.org/schema/configuration/searchIndexMappings.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "SearchIndexMappings", + "description": "Admin-editable Elasticsearch/OpenSearch index mappings, persisted in settings and keyed by language and entity type. The stored mapping is the effective mapping used when an index is (re)created; it already carries the field-safety guards (ignore_above, ignore_malformed, mapping limits) baked in at seed time.", + "type": "object", + "javaType": "org.openmetadata.schema.configuration.SearchIndexMappings", + "properties": { + "languages": { + "description": "Mappings keyed by search index mapping language (e.g. 'en', 'jp', 'ru', 'zh'), then by entity type (e.g. 'table', 'topic'). Each leaf value is the raw index mapping document.", + "type": "object", + "existingJavaType": "java.util.Map>", + "additionalProperties": true + } + }, + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/settings/settings.json b/openmetadata-spec/src/main/resources/json/schema/settings/settings.json index 350808764455..3c5d4cfacded 100644 --- a/openmetadata-spec/src/main/resources/json/schema/settings/settings.json +++ b/openmetadata-spec/src/main/resources/json/schema/settings/settings.json @@ -42,7 +42,8 @@ "openLineageSettings", "mcpConfiguration", "glossaryTermRelationSettings", - "aiSettings" + "aiSettings", + "searchIndexMappings" ] } }, @@ -121,6 +122,9 @@ }, { "$ref": "../configuration/aiSettings.json" + }, + { + "$ref": "../configuration/searchIndexMappings.json" } ] } diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchIndexNestedColumns.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchIndexNestedColumns.spec.ts index ef8609dd511b..0648981b78fd 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchIndexNestedColumns.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchIndexNestedColumns.spec.ts @@ -40,6 +40,10 @@ const buildOversizedExpression = (): string => { const table = new TableClass(); const leafColumnName = `deepleaf${lettersToken()}`; +// A shallow, within-depth-limit column with a letters-only name. Columns deeper than +// index.mapping.depth.limit (the 25-level leaf above) are intentionally dropped from +// columnNamesFuzzy; that limit is tunable via the search-index-mappings API. +const searchableColumnName = `findablecol${lettersToken()}`; // A struct column nested NESTING_DEPTH levels deep (past index.mapping.depth.limit) whose deepest // leaf carries the oversized expression — the combined depth + immense-term worst case. @@ -67,8 +71,15 @@ const buildDeeplyNestedOversizedColumn = (): Column => { test.describe('Search index - deeply nested oversized columns', () => { test.beforeAll(async ({ browser }) => { const { apiContext, afterAction } = await createNewPage(browser); + const searchableColumn = { + name: searchableColumnName, + dataType: DataType.Varchar, + dataLength: 64, + dataTypeDisplay: 'varchar(64)', + } as Column; table.entity.columns = [ buildDeeplyNestedOversizedColumn(), + searchableColumn, ...table.entity.columns, ]; await table.create(apiContext); @@ -106,7 +117,7 @@ test.describe('Search index - deeply nested oversized columns', () => { await redirectToHomePage(page); }); - test('25-level oversized nested column indexes and is searchable by its deep column name', async ({ + test('oversized deeply nested column indexes and an in-limit column name is searchable', async ({ page, }) => { const searchInput = page.getByTestId('searchBox'); @@ -123,11 +134,12 @@ test.describe('Search index - deeply nested oversized columns', () => { await expect(suggestions).toBeVisible(); await expect(suggestions).toContainText(table.entity.name); - // Searching: the 25-level-deep column name surfaces the table via columnNamesFuzzy - the - // mechanism that replaced the dropped flattened columns.children.name search field. + // Searching: an in-depth-limit column name surfaces the table via columnNamesFuzzy - the + // mechanism that replaced the dropped flattened columns.children.name search field. The + // 25-level-deep leaf is past the default depth limit and is intentionally not indexed here. await searchInput.clear(); const byColumnResponse = page.waitForResponse('/api/v1/search/query?*'); - await searchInput.fill(leafColumnName); + await searchInput.fill(searchableColumnName); await byColumnResponse; await expect(suggestions).toBeVisible(); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/SettingsRouter.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/SettingsRouter.tsx index b4ea99ae66eb..e0ed194d9a48 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/SettingsRouter.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/SettingsRouter.tsx @@ -252,6 +252,15 @@ const AISettingsPage = withSuspenseFallback( React.lazy(() => import('../../pages/AISettingsPage/AISettingsPage')) ); +const SearchIndexMappingsPage = withSuspenseFallback( + React.lazy( + () => + import( + '../../pages/SearchIndexMappingsPage/SearchIndexMappingsPage.component' + ) + ) +); + const SearchSettingsPage = withSuspenseFallback( React.lazy(() => import('../../pages/SearchSettingsPage/SearchSettingsPage')) ); @@ -704,6 +713,18 @@ const SettingsRouter = () => { )} /> + + + + } + path={getSettingPathRelative( + GlobalSettingsMenuCategory.PREFERENCES, + GlobalSettingOptions.SEARCH_MAPPINGS + )} + /> + diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/GlobalSettings.constants.ts b/openmetadata-ui/src/main/resources/ui/src/constants/GlobalSettings.constants.ts index c1c586ccdaea..fea244ec1cbf 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/GlobalSettings.constants.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/GlobalSettings.constants.ts @@ -85,6 +85,7 @@ export enum GlobalSettingOptions { OM_URL_CONFIG = 'om-url-config', AI_SETTINGS = 'ai-settings', SEARCH_SETTINGS = 'search-settings', + SEARCH_MAPPINGS = 'search-mappings', DATA_ASSETS = 'dataAssets', QUERY = 'query', TEST_CASES = 'testCases', diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/PageHeaders.constant.ts b/openmetadata-ui/src/main/resources/ui/src/constants/PageHeaders.constant.ts index 80811e0985e9..07950ea07423 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/PageHeaders.constant.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/PageHeaders.constant.ts @@ -212,6 +212,10 @@ export const PAGE_HEADERS = { header: 'label.search', subHeader: 'message.page-sub-header-for-search-setting', }, + SEARCH_INDEX_MAPPINGS: { + header: 'label.search-mapping-plural', + subHeader: 'message.page-sub-header-for-search-index-mappings', + }, LINEAGE_CONFIG: { header: 'label.lineage-config', subHeader: 'message.page-sub-header-for-lineage-config-setting', diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/configuration/elasticSearchConfiguration.ts b/openmetadata-ui/src/main/resources/ui/src/generated/configuration/elasticSearchConfiguration.ts index 9ae2b7b3fa39..52597497c48e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/configuration/elasticSearchConfiguration.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/configuration/elasticSearchConfiguration.ts @@ -73,7 +73,13 @@ export interface ElasticSearchConfiguration { * Index factory name */ searchIndexFactoryClassName?: string; - searchIndexMappingLanguage: SearchIndexMappingLanguage; + /** + * Limits applied while building search documents so that field values can never be rejected + * by Elasticsearch/OpenSearch. Values default to the documented engine defaults; override + * to tune without changing infrastructure settings. + */ + searchIndexingLimits?: SearchIndexingLimits; + searchIndexMappingLanguage: SearchIndexMappingLanguage; /** * This enum defines the search Type elastic/open search. */ @@ -219,6 +225,44 @@ export enum SearchIndexMappingLanguage { Zh = "ZH", } +/** + * Limits applied while building search documents so that field values can never be rejected + * by Elasticsearch/OpenSearch. Values default to the documented engine defaults; override + * to tune without changing infrastructure settings. + */ +export interface SearchIndexingLimits { + /** + * Enable injecting ignore_above / ignore_malformed and index.mapping.*.limit guardrails + * into index mappings at creation time so documents cannot be rejected. When false, + * mappings are created as-is. + */ + enableMappingHardening?: boolean; + /** + * Maximum UTF-8 byte length of a single keyword term. ignore_above is set to a byte-safe + * character count derived from this (value/4). The hard Lucene limit is 32766 bytes. + */ + keywordMaxBytes?: number; + /** + * Maximum object/column nesting depth. Mirrors index.mapping.depth.limit. + */ + mappingDepthLimit?: number; + /** + * Maximum number of flattened columns or schema fields indexed for a single data asset. + * Items beyond this are dropped from the search document. + */ + maxColumns?: number; + /** + * Maximum number of nested-type objects allowed in a single document before + * Elasticsearch/OpenSearch rejects it (the engine rejects rather than truncates). Mirrors + * index.mapping.nested_objects.limit. + */ + nestedObjectsLimit?: number; + /** + * Maximum total fields per index. Mirrors index.mapping.total_fields.limit. + */ + totalFieldsLimit?: number; +} + /** * This enum defines the search Type elastic/open search. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/configuration/searchIndexMappings.ts b/openmetadata-ui/src/main/resources/ui/src/generated/configuration/searchIndexMappings.ts new file mode 100644 index 000000000000..f8a292ef8779 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/configuration/searchIndexMappings.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Admin-editable Elasticsearch/OpenSearch index mappings, persisted in settings and keyed + * by language and entity type. The stored mapping is the effective mapping used when an + * index is (re)created; it already carries the field-safety guards (ignore_above, + * ignore_malformed, mapping limits) baked in at seed time. + */ +export interface SearchIndexMappings { + /** + * Mappings keyed by search index mapping language (e.g. 'en', 'jp', 'ru', 'zh'), then by + * entity type (e.g. 'table', 'topic'). Each leaf value is the raw index mapping document. + */ + languages?: { [key: string]: any }; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/settings/settings.ts b/openmetadata-ui/src/main/resources/ui/src/generated/settings/settings.ts index 86cbbb7395c2..b24a1161e53e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/settings/settings.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/settings/settings.ts @@ -49,6 +49,7 @@ export enum SettingType { ProfilerConfiguration = "profilerConfiguration", SandboxModeEnabled = "sandboxModeEnabled", ScimConfiguration = "scimConfiguration", + SearchIndexMappings = "searchIndexMappings", SearchSettings = "searchSettings", SecretsManagerConfiguration = "secretsManagerConfiguration", SecurityConfiguration = "securityConfiguration", @@ -112,6 +113,11 @@ export enum SettingType { * * Configuration for AI features: memory extraction, the Memory Agent, and tunable LLM * system prompts. + * + * Admin-editable Elasticsearch/OpenSearch index mappings, persisted in settings and keyed + * by language and entity type. The stored mapping is the effective mapping used when an + * index is (re)created; it already carries the field-safety guards (ignore_above, + * ignore_malformed, mapping limits) baked in at seed time. */ export interface PipelineServiceClientConfiguration { /** @@ -377,7 +383,13 @@ export interface PipelineServiceClientConfiguration { * Index factory name */ searchIndexFactoryClassName?: string; - searchIndexMappingLanguage?: SearchIndexMappingLanguage; + /** + * Limits applied while building search documents so that field values can never be rejected + * by Elasticsearch/OpenSearch. Values default to the documented engine defaults; override + * to tune without changing infrastructure settings. + */ + searchIndexingLimits?: SearchIndexingLimits; + searchIndexMappingLanguage?: SearchIndexMappingLanguage; /** * This enum defines the search Type elastic/open search. */ @@ -639,6 +651,11 @@ export interface PipelineServiceClientConfiguration { memoryAgent?: MemoryAgent; memoryExtraction?: MemoryExtraction; prompts?: Prompts; + /** + * Mappings keyed by search index mapping language (e.g. 'en', 'jp', 'ru', 'zh'), then by + * entity type (e.g. 'table', 'topic'). Each leaf value is the raw index mapping document. + */ + languages?: { [key: string]: any }; } export interface AllowedFieldValueBoostFields { @@ -2626,6 +2643,44 @@ export enum SearchIndexMappingLanguage { Zh = "ZH", } +/** + * Limits applied while building search documents so that field values can never be rejected + * by Elasticsearch/OpenSearch. Values default to the documented engine defaults; override + * to tune without changing infrastructure settings. + */ +export interface SearchIndexingLimits { + /** + * Enable injecting ignore_above / ignore_malformed and index.mapping.*.limit guardrails + * into index mappings at creation time so documents cannot be rejected. When false, + * mappings are created as-is. + */ + enableMappingHardening?: boolean; + /** + * Maximum UTF-8 byte length of a single keyword term. ignore_above is set to a byte-safe + * character count derived from this (value/4). The hard Lucene limit is 32766 bytes. + */ + keywordMaxBytes?: number; + /** + * Maximum object/column nesting depth. Mirrors index.mapping.depth.limit. + */ + mappingDepthLimit?: number; + /** + * Maximum number of flattened columns or schema fields indexed for a single data asset. + * Items beyond this are dropped from the search document. + */ + maxColumns?: number; + /** + * Maximum number of nested-type objects allowed in a single document before + * Elasticsearch/OpenSearch rejects it (the engine rejects rather than truncates). Mirrors + * index.mapping.nested_objects.limit. + */ + nestedObjectsLimit?: number; + /** + * Maximum total fields per index. Mirrors index.mapping.total_fields.limit. + */ + totalFieldsLimit?: number; +} + /** * This enum defines the search Type elastic/open search. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ar-sa.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ar-sa.json index d6b8992753e3..6285ace0c7b4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ar-sa.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ar-sa.json @@ -1988,6 +1988,7 @@ "reset-default-layout": "إعادة تعيين التخطيط الافتراضي", "reset-entity": "إعادة تعيين {{entity}}", "reset-position": "إعادة تعيين إلى المركز", + "reset-to-default": "Reset to default", "reset-view": "إعادة تعيين العرض", "reset-your-password": "إعادة تعيين كلمة المرور الخاصة بك", "resolution": "القرار", @@ -2120,6 +2121,8 @@ "search-index-plural": "فهارس البحث", "search-index-setting-plural": "إعدادات فهرس البحث", "search-insights": "رؤى البحث", + "search-mapping": "Search Mapping", + "search-mapping-plural": "Search Mappings", "search-memories": "البحث في الذكريات حسب المحتوى...", "search-rbac": "البحث RBAC", "search-result-plural": "نتائج البحث", @@ -3266,6 +3269,7 @@ "invalid-dimensions": "أبعاد الصورة غير صالحة", "invalid-file": "ملف غير صالح", "invalid-file-format": "تنسيق ملف غير صالح. الصيغ المقبولة: {{formats}}", + "invalid-json-mapping": "Invalid JSON. Please fix the mapping before saving.", "invalid-object-key": "مفتاح كائن غير صالح. يجب أن يبدأ بحرف، شرطة سفلية، أو علامة الدولار، متبوعًا بأحرف، شرطات سفلية، علامات الدولار، أو أرقام.", "invalid-odcs-contract-format": "تنسيق عقد ODCS غير صالح. الحقول المطلوبة: apiVersion، kind، status.", "invalid-odcs-contract-format-required-fields": "الحقول المطلوبة: APIVersion, Kind, Status", @@ -3580,6 +3584,7 @@ "page-sub-header-for-profiler-configuration": "خصص سلوك المُحلل (profiler) عالميًا عن طريق تعيين المقاييس التي يجب حسابها بناءً على أنواع بيانات الأعمدة", "page-sub-header-for-roles": "تعيين وصول شامل قائم على الدور للمستخدمين أو الفرق.", "page-sub-header-for-search": "استيعاب البيانات الوصفية من خدمات البحث الأكثر شيوعًا.", + "page-sub-header-for-search-index-mappings": "View and edit the Elasticsearch/OpenSearch index mappings for each entity type and language.", "page-sub-header-for-search-setting": "القدرة على تكوين إعدادات البحث لتناسب احتياجاتك.", "page-sub-header-for-security": "استيعاب البيانات الوصفية من خدمات الأمان الأكثر شيوعًا.", "page-sub-header-for-setting": "القدرة على تكوين تطبيق {{brandName}} ليناسب احتياجاتك.", @@ -3690,6 +3695,7 @@ "search-for-edge": "البحث عن مسار (Pipeline)، إجراءات مخزنة (StoredProcedures)", "search-for-entity-types": "البحث عن الجداول، المواضيع، لوحات المعلومات، المسارات، نماذج تعلم الآلة (ML Models)، المسرد والعلامات.", "search-for-ingestion": "البحث عن الاستيعاب", + "search-index-mappings-reindex-info": "Changes to a mapping are saved to settings and applied on the next reindex of that entity. Run a reindex after saving for changes to take effect.", "search-insights-description": "اعرض إحصائيات عنقود البحث وتفاصيل الفهرس وأدر الفهارس اليتيمة.", "search-settings-description": "إعدادات البحث لكل كيان بناءً على التصنيفات.", "search-settings-for-entity": "إعدادات البحث بناءً على التصنيف لـ {{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json index 4f580a36b4d0..1b96bdba0aca 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json @@ -1988,6 +1988,7 @@ "reset-default-layout": "Standard Layout zurücksetzen", "reset-entity": "{{entity}} zurücksetzen", "reset-position": "Auf Mitte zurücksetzen", + "reset-to-default": "Reset to default", "reset-view": "Ansicht zurücksetzen", "reset-your-password": "Ihr Passwort zurücksetzen", "resolution": "Auflösung", @@ -2120,6 +2121,8 @@ "search-index-plural": "Suchindizes", "search-index-setting-plural": "Suchindexeinstellungen", "search-insights": "Sucheinblicke", + "search-mapping": "Search Mapping", + "search-mapping-plural": "Search Mappings", "search-memories": "Erinnerungen nach Inhalt durchsuchen...", "search-rbac": "RBAC durchsuchen", "search-result-plural": "Suchergebnisse", @@ -3266,6 +3269,7 @@ "invalid-dimensions": "Ungültige Bildabmessungen", "invalid-file": "Ungültige Datei", "invalid-file-format": "Ungültiges Dateiformat. Akzeptierte Formate: {{formats}}", + "invalid-json-mapping": "Invalid JSON. Please fix the mapping before saving.", "invalid-object-key": "Ungültiger Objektschlüssel. Muss mit einem Buchstaben, einem Unterstrich oder einem Dollarzeichen beginnen, gefolgt von Buchstaben, Unterstrichen, Dollarzeichen oder Ziffern.", "invalid-odcs-contract-format": "Ungültiges ODCS-Vertragsformat. Erforderliche Felder: apiVersion, kind, status.", "invalid-odcs-contract-format-required-fields": "Die folgenden Felder sind erforderlich: APIVersion, Kind, Status", @@ -3580,6 +3584,7 @@ "page-sub-header-for-profiler-configuration": "Passen Sie das Verhalten des Profilers global an, indem Sie die zu berechnenden Metriken basierend auf den Spaltendatentypen festlegen.", "page-sub-header-for-roles": "Weise Benutzern oder Teams umfassende rollenbasierte Zugriffsberechtigungen zu.", "page-sub-header-for-search": "Ingestion von Metadaten aus den beliebtesten Suchdiensten.", + "page-sub-header-for-search-index-mappings": "View and edit the Elasticsearch/OpenSearch index mappings for each entity type and language.", "page-sub-header-for-search-setting": "Möglichkeit, die Sucheinstellungen an Ihre Bedürfnisse anzupassen.", "page-sub-header-for-security": "Ingestion von Metadaten aus den beliebtesten Sicherheitsdiensten.", "page-sub-header-for-setting": "Möglichkeit, die {{brandName}}-Anwendung an Ihre Anforderungen anzupassen.", @@ -3690,6 +3695,7 @@ "search-for-edge": "Suche nach Pipeline, StoredProcedures", "search-for-entity-types": "Suche nach Tabellen, Themen, Dashboards, Pipelines, ML-Modellen, Glossar und Tags.", "search-for-ingestion": "Suche nach Ingestion", + "search-index-mappings-reindex-info": "Changes to a mapping are saved to settings and applied on the next reindex of that entity. Run a reindex after saving for changes to take effect.", "search-insights-description": "Zeigen Sie Suchclusterstatistiken und Indexdetails an und verwalten Sie verwaiste Indizes.", "search-settings-description": "Sucheinstellungen für jede Entität basierend auf Rankings.", "search-settings-for-entity": "Sucheinstellungen basierend auf Ranking für {{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json index 7f1452433359..975d7f4d9b52 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json @@ -1988,6 +1988,7 @@ "reset-default-layout": "Reset Default Layout", "reset-entity": "Reset {{entity}}", "reset-position": "Reset to Center", + "reset-to-default": "Reset to default", "reset-view": "Reset view", "reset-your-password": "Reset your Password", "resolution": "Resolution", @@ -2120,6 +2121,8 @@ "search-index-plural": "Search Indexes", "search-index-setting-plural": "Search Index Settings", "search-insights": "Search Insights", + "search-mapping": "Search Mapping", + "search-mapping-plural": "Search Mappings", "search-memories": "Search memories by content...", "search-rbac": "Search RBAC", "search-result-plural": "Search Results", @@ -3266,6 +3269,7 @@ "invalid-dimensions": "Invalid image dimensions", "invalid-file": "Invalid file", "invalid-file-format": "Invalid file format. Accepted formats: {{formats}}", + "invalid-json-mapping": "Invalid JSON. Please fix the mapping before saving.", "invalid-object-key": "Invalid object key. Must start with a letter, underscore, or dollar sign, followed by letters, underscores, dollar signs, or digits.", "invalid-odcs-contract-format": "Invalid ODCS contract format. Required fields: apiVersion, kind, status.", "invalid-odcs-contract-format-required-fields": "The following fields are required: APIVersion, Kind, Status", @@ -3580,6 +3584,7 @@ "page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types", "page-sub-header-for-roles": "Assign comprehensive role based access to Users or Teams.", "page-sub-header-for-search": "Ingest metadata from the most popular search services.", + "page-sub-header-for-search-index-mappings": "View and edit the Elasticsearch/OpenSearch index mappings for each entity type and language.", "page-sub-header-for-search-setting": "Ability to configure the search settings to suit your needs.", "page-sub-header-for-security": "Ingest metadata from the most popular security services.", "page-sub-header-for-setting": "Ability to configure the {{brandName}} application to suit your needs.", @@ -3690,6 +3695,7 @@ "search-for-edge": "Search for Pipeline, StoredProcedures", "search-for-entity-types": "Search for Tables, Topics, Dashboards, Pipelines, ML Models, Glossary and Tags.", "search-for-ingestion": "Search for ingestion", + "search-index-mappings-reindex-info": "Changes to a mapping are saved to settings and applied on the next reindex of that entity. Run a reindex after saving for changes to take effect.", "search-insights-description": "View search cluster statistics, index details, and manage orphan indexes.", "search-settings-description": "Search settings for each entity based on rankings.", "search-settings-for-entity": "Search settings based on ranking for {{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json index 976fc672ae7b..cae381c5d9b8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json @@ -1988,6 +1988,7 @@ "reset-default-layout": "Restablecer el panel por defecto", "reset-entity": "Restablecer {{entity}}", "reset-position": "Restablecer al centro", + "reset-to-default": "Reset to default", "reset-view": "Restablecer vista", "reset-your-password": "Restablezca su contraseña", "resolution": "Resolución", @@ -2120,6 +2121,8 @@ "search-index-plural": "Índices de Búsqueda", "search-index-setting-plural": "Configuración de Índices de Búsqueda", "search-insights": "Información de búsqueda", + "search-mapping": "Search Mapping", + "search-mapping-plural": "Search Mappings", "search-memories": "Buscar recuerdos por contenido...", "search-rbac": "Buscar RBAC", "search-result-plural": "Resultados de búsqueda", @@ -3266,6 +3269,7 @@ "invalid-dimensions": "Dimensiones de imagen inválidas", "invalid-file": "Archivo inválido", "invalid-file-format": "Formato de archivo inválido. Formatos aceptados: {{formats}}", + "invalid-json-mapping": "Invalid JSON. Please fix the mapping before saving.", "invalid-object-key": "Clave de objeto no válida. Debe comenzar con una letra, un guion bajo o un signo de dólar, seguido de letras, guiones bajos, signos de dólar o dígitos.", "invalid-odcs-contract-format": "Formato de contrato ODCS inválido. Campos requeridos: apiVersion, kind, status.", "invalid-odcs-contract-format-required-fields": "Los siguientes campos son requeridos: APIVersion, Kind, Status", @@ -3580,6 +3584,7 @@ "page-sub-header-for-profiler-configuration": "Personaliza a nivel global el comportamiento del perfilador ajustando las métricas a calcular basadas en los tipos de datos de las columnas", "page-sub-header-for-roles": "Asigna un acceso basado en roles integral a Usuarios o Equipos.", "page-sub-header-for-search": "Ingresa metadatos desde los servicios de búsqueda más populares.", + "page-sub-header-for-search-index-mappings": "View and edit the Elasticsearch/OpenSearch index mappings for each entity type and language.", "page-sub-header-for-search-setting": "Capacidad de configurar los ajustes de búsqueda en base a tus necesidades.", "page-sub-header-for-security": "Ingresa metadatos desde los servicios de seguridad más populares.", "page-sub-header-for-setting": "Capacidad para configurar la aplicación {{brandName}} según tus necesidades.", @@ -3690,6 +3695,7 @@ "search-for-edge": "Buscar Pipeline, Procedimientos Almacenados", "search-for-entity-types": "Buscar Tablas, Temas, Paneles, Pipelines, Modelos de ML, Glosarios y Etiquetas.", "search-for-ingestion": "Buscar orígenes de datos", + "search-index-mappings-reindex-info": "Changes to a mapping are saved to settings and applied on the next reindex of that entity. Run a reindex after saving for changes to take effect.", "search-insights-description": "Vea estadísticas del grupo de búsqueda, detalles del índice y administre índices huérfanos.", "search-settings-description": "Configuración de búsqueda para cada entidad basada en rankings.", "search-settings-for-entity": "Configuración de búsqueda basada en ranking para {{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json index c947d8f99f62..a4892aba7e25 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json @@ -1988,6 +1988,7 @@ "reset-default-layout": "Réinitialiser la Disposition par Défaut", "reset-entity": "Réinitialiser {{entity}}", "reset-position": "Réinitialiser au centre", + "reset-to-default": "Reset to default", "reset-view": "Réinitialiser la vue", "reset-your-password": "Réinitialiser le Mot de Passe", "resolution": "Résolution", @@ -2120,6 +2121,8 @@ "search-index-plural": "Indexes de Recherche", "search-index-setting-plural": "Paramètres des Index de Recherche", "search-insights": "Statistiques de recherche", + "search-mapping": "Search Mapping", + "search-mapping-plural": "Search Mappings", "search-memories": "Rechercher des souvenirs par contenu...", "search-rbac": "Rechercher RBAC", "search-result-plural": "Résultats de recherche", @@ -3266,6 +3269,7 @@ "invalid-dimensions": "Dimensions d'image non valides", "invalid-file": "Fichier non valide", "invalid-file-format": "Format de fichier non valide. Formats acceptés : {{formats}}", + "invalid-json-mapping": "Invalid JSON. Please fix the mapping before saving.", "invalid-object-key": "Clé d'objet non valide. Doit commencer par une lettre, un underscore ou signe dollar, suivi de lettres, underscores, signes dollars, ou chiffres.", "invalid-odcs-contract-format": "Format de contrat ODCS non valide. Champs requis : apiVersion, kind, status.", "invalid-odcs-contract-format-required-fields": "Les champs suivants sont requis: APIVersion, Kind, Status", @@ -3580,6 +3584,7 @@ "page-sub-header-for-profiler-configuration": "Personnalisez le comportement global du profiler en établissant les métriques à calculer à partir des types de données des colonnes", "page-sub-header-for-roles": "Attribuez des autorisations basées sur les rôles aux utilisateurs ou aux équipes.", "page-sub-header-for-search": "Ingestion de métadonnées à partir des services de recherche les plus populaires.", + "page-sub-header-for-search-index-mappings": "View and edit the Elasticsearch/OpenSearch index mappings for each entity type and language.", "page-sub-header-for-search-setting": "Capacité de configurer les paramètres de recherche selon vos besoins.", "page-sub-header-for-security": "Ingestion de métadonnées à partir des services de sécurité les plus populaires.", "page-sub-header-for-setting": "Configurez l'application {{brandName}} pour répondre à vos besoins.", @@ -3690,6 +3695,7 @@ "search-for-edge": "Rechercher Pipeline, Procédures Stockées", "search-for-entity-types": "Rechercher Tables, Topics, Tableaux de Bord, Pipelines et Modèles d'IA", "search-for-ingestion": "Rechercher une ingestion", + "search-index-mappings-reindex-info": "Changes to a mapping are saved to settings and applied on the next reindex of that entity. Run a reindex after saving for changes to take effect.", "search-insights-description": "Affichez les statistiques du cluster de recherche, les détails de l'index et gérez les index orphelins.", "search-settings-description": "Paramètres de recherche pour chaque entité basés sur les classements.", "search-settings-for-entity": "Paramètres de recherche basés sur le classement pour {{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/gl-es.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/gl-es.json index efdc2fe7e33a..31fd9e70670c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/gl-es.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/gl-es.json @@ -1988,6 +1988,7 @@ "reset-default-layout": "Restablecer deseño predeterminado", "reset-entity": "Restablecer {{entity}}", "reset-position": "Restablecer ao centro", + "reset-to-default": "Reset to default", "reset-view": "Restablecer vista", "reset-your-password": "Restablecer o teu contrasinal", "resolution": "Resolución", @@ -2120,6 +2121,8 @@ "search-index-plural": "Índices de busca", "search-index-setting-plural": "Axustes do índice de busca", "search-insights": "Insights de busca", + "search-mapping": "Search Mapping", + "search-mapping-plural": "Search Mappings", "search-memories": "Buscar recordos por contido...", "search-rbac": "Búsqueda RBAC", "search-result-plural": "Resultados da busca", @@ -3266,6 +3269,7 @@ "invalid-dimensions": "Dimensións de imaxe inválidas", "invalid-file": "Ficheiro inválido", "invalid-file-format": "Formato de ficheiro inválido. Formatos aceptados: {{formats}}", + "invalid-json-mapping": "Invalid JSON. Please fix the mapping before saving.", "invalid-object-key": "Clave de obxecto non válida. Debe comezar cunha letra, subliñado ou símbolo de dólar, seguido de letras, subliñados, signos de dólar ou díxitos.", "invalid-odcs-contract-format": "Formato de contrato ODCS inválido. Campos requiridos: apiVersion, kind, status.", "invalid-odcs-contract-format-required-fields": "Os seguintes campos son requiridos: APIVersion, Kind, Status", @@ -3580,6 +3584,7 @@ "page-sub-header-for-profiler-configuration": "Personaliza globalmente o comportamento do perfilador definindo as métricas a calcular baseadas nos tipos de datos das columnas", "page-sub-header-for-roles": "Asigna un acceso baseado en roles detallados a Usuarios ou Equipos.", "page-sub-header-for-search": "Inxesta metadatos dos servizos de busca máis populares.", + "page-sub-header-for-search-index-mappings": "View and edit the Elasticsearch/OpenSearch index mappings for each entity type and language.", "page-sub-header-for-search-setting": "Capacidade de configurar os axustes de busca segundo as súas necesidades.", "page-sub-header-for-security": "Inxesta metadatos dos servizos de seguridade máis populares.", "page-sub-header-for-setting": "Posibilidade de configurar a aplicación {{brandName}} para adaptala ás túas necesidades.", @@ -3690,6 +3695,7 @@ "search-for-edge": "Buscar Pipeline, Procedementos Almacenados", "search-for-entity-types": "Buscar Táboas, Temas, Paneis, Pipelines, Modelos de ML, Glosarios e Etiquetas.", "search-for-ingestion": "Buscar inxestión", + "search-index-mappings-reindex-info": "Changes to a mapping are saved to settings and applied on the next reindex of that entity. Run a reindex after saving for changes to take effect.", "search-insights-description": "Visualiza estatísticas do clúster de busca, detalles de índices e xestiona índices orfos.", "search-settings-description": "Configuración de busca para cada entidade baseada en clasificacións.", "search-settings-for-entity": "Configuración de busca baseada en clasificación para {{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json index b335e4b182c9..14c22a543f16 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json @@ -1988,6 +1988,7 @@ "reset-default-layout": "איפוס פריסת ברירת מחדל", "reset-entity": "איפוס {{entity}}", "reset-position": "אפס למרכז", + "reset-to-default": "Reset to default", "reset-view": "איפוס תצוגה", "reset-your-password": "איפוס הסיסמה שלך", "resolution": "רזולוציה", @@ -2120,6 +2121,8 @@ "search-index-plural": "אינדקסי חיפוש", "search-index-setting-plural": "הגדרות אינדקס חיפוש", "search-insights": "תובנות חיפוש", + "search-mapping": "Search Mapping", + "search-mapping-plural": "Search Mappings", "search-memories": "חפש זיכרונות לפי תוכן...", "search-rbac": "חפש RBAC", "search-result-plural": "תוצאות חיפוש", @@ -3266,6 +3269,7 @@ "invalid-dimensions": "מידות תמונה לא חוקיות", "invalid-file": "קובץ לא חוקי", "invalid-file-format": "פורמט קובץ לא חוקי. פורמטים מקובלים: {{formats}}", + "invalid-json-mapping": "Invalid JSON. Please fix the mapping before saving.", "invalid-object-key": "מפתח אובייקט לא חוקי. חייב להתחיל עם אות, קו תחתון או סימן דולר, ולפניו אות, קו תחתון או סימן דולר או ספרות.", "invalid-odcs-contract-format": "פורמט חוזה ODCS לא חוקי. שדות נדרשים: apiVersion, kind, status.", "invalid-odcs-contract-format-required-fields": "השדות הבאים נדרשים: APIVersion, Kind, Status", @@ -3580,6 +3584,7 @@ "page-sub-header-for-profiler-configuration": "התאם אישית גלובלית את התנהגות הפרופיילר על ידי הגדרת המדדים לחישוב על בסיס סוגי נתוני עמודות", "page-sub-header-for-roles": "הקצאת גישה מבוססת תפקיד למשתמשים או קבוצות.", "page-sub-header-for-search": "שלב מטה-דאטה משירותי החיפוש הפופולריים ביותר.", + "page-sub-header-for-search-index-mappings": "View and edit the Elasticsearch/OpenSearch index mappings for each entity type and language.", "page-sub-header-for-search-setting": "יכולת להגדיר את הגדרות החיפוש לפי צרכיך.", "page-sub-header-for-security": "שלב מטה-דאטה משירותי האבטחה הפופולריים ביותר.", "page-sub-header-for-setting": "יכולת להגדיר את יישום {{brandName}} לפי צרכיך.", @@ -3690,6 +3695,7 @@ "search-for-edge": "חיפוש לפי תהליך טעינה, פונקציות מאוחסנות", "search-for-entity-types": "חיפוש לפי טבלאות, נושאים, דשבורדים, תהליכי טעינה, מודלי למידת מכונה, מילוני מונחים ותגיות", "search-for-ingestion": "חיפוש תהליך טעינה (Ingestion)", + "search-index-mappings-reindex-info": "Changes to a mapping are saved to settings and applied on the next reindex of that entity. Run a reindex after saving for changes to take effect.", "search-insights-description": "הצג סטטיסטיקות אשכול חיפוש, פרטי אינדקס ונהל אינדקסים יתומים.", "search-settings-description": "הגדרות חיפוש לכל ישות על בסיס דירוגים.", "search-settings-for-entity": "הגדרות חיפוש מבוססות דירוג עבור {{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json index 0444c032cc09..3fa8dd61bcb2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json @@ -1988,6 +1988,7 @@ "reset-default-layout": "レイアウトをデフォルトにリセット", "reset-entity": "{{entity}}をリセット", "reset-position": "中央にリセット", + "reset-to-default": "Reset to default", "reset-view": "表示をリセット", "reset-your-password": "パスワードをリセット", "resolution": "解決", @@ -2120,6 +2121,8 @@ "search-index-plural": "検索インデックス", "search-index-setting-plural": "検索インデックス設定", "search-insights": "検索インサイト", + "search-mapping": "Search Mapping", + "search-mapping-plural": "Search Mappings", "search-memories": "内容でメモリを検索...", "search-rbac": "RBAC 検索", "search-result-plural": "検索結果", @@ -3266,6 +3269,7 @@ "invalid-dimensions": "無効な画像サイズ", "invalid-file": "無効なファイル", "invalid-file-format": "無効なファイル形式。対応形式：{{formats}}", + "invalid-json-mapping": "Invalid JSON. Please fix the mapping before saving.", "invalid-object-key": "無効なオブジェクトキーです。英字、アンダースコア（_）、ドル記号（$）で始まり、それに英数字、アンダースコア、ドル記号が続く必要があります。", "invalid-odcs-contract-format": "無効な ODCS 契約形式です。必須フィールド：apiVersion、kind、status。", "invalid-odcs-contract-format-required-fields": "以下のフィールドが必須です: APIVersion, Kind, Status", @@ -3580,6 +3584,7 @@ "page-sub-header-for-profiler-configuration": "カラムのデータ型に基づいて計算されるメトリクスを設定し、プロファイラーの動作をグローバルにカスタマイズします。", "page-sub-header-for-roles": "ユーザーやチームに包括的なロールベースのアクセスを割り当てます。", "page-sub-header-for-search": "人気の検索サービスからメタデータを取り込みます。", + "page-sub-header-for-search-index-mappings": "View and edit the Elasticsearch/OpenSearch index mappings for each entity type and language.", "page-sub-header-for-search-setting": "ニーズに合わせて検索設定を構成します。", "page-sub-header-for-security": "最も人気のあるセキュリティサービスからメタデータを取り込みます。", "page-sub-header-for-setting": "{{brandName}} アプリケーションをニーズに合わせて構成します。", @@ -3690,6 +3695,7 @@ "search-for-edge": "パイプライン、ストアドプロシージャを検索", "search-for-entity-types": "テーブル、トピック、ダッシュボード、パイプライン、MLモデルを検索", "search-for-ingestion": "取り込みを検索", + "search-index-mappings-reindex-info": "Changes to a mapping are saved to settings and applied on the next reindex of that entity. Run a reindex after saving for changes to take effect.", "search-insights-description": "検索クラスターの統計、インデックスの詳細を表示し、孤立したインデックスを管理します。", "search-settings-description": "各エンティティのランキングに基づく検索設定。", "search-settings-for-entity": "{{entity}}の検索設定（ランキングに基づく）", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ko-kr.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ko-kr.json index cede8fe2ec7b..6cea89b91eab 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ko-kr.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ko-kr.json @@ -1988,6 +1988,7 @@ "reset-default-layout": "기본 레이아웃 초기화", "reset-entity": "{{entity}} 초기화", "reset-position": "중앙으로 재설정", + "reset-to-default": "Reset to default", "reset-view": "보기 초기화", "reset-your-password": "비밀번호 재설정", "resolution": "해결", @@ -2120,6 +2121,8 @@ "search-index-plural": "검색 인덱스들", "search-index-setting-plural": "검색 인덱스 설정들", "search-insights": "검색 인사이트", + "search-mapping": "Search Mapping", + "search-mapping-plural": "Search Mappings", "search-memories": "내용으로 메모리 검색...", "search-rbac": "RBAC 검색", "search-result-plural": "검색 결과", @@ -3266,6 +3269,7 @@ "invalid-dimensions": "잘못된 이미지 크기", "invalid-file": "잘못된 파일", "invalid-file-format": "잘못된 파일 형식입니다. 허용되는 형식: {{formats}}", + "invalid-json-mapping": "Invalid JSON. Please fix the mapping before saving.", "invalid-object-key": "유효하지 않은 객체 키입니다. 문자, 밑줄 또는 달러 기호로 시작하고 그 뒤에 문자, 밑줄, 달러 기호 또는 숫자가 와야 합니다.", "invalid-odcs-contract-format": "잘못된 ODCS 계약 형식입니다. 필수 필드: apiVersion, kind, status.", "invalid-odcs-contract-format-required-fields": "다음 필드가 필요합니다: APIVersion, Kind, Status", @@ -3580,6 +3584,7 @@ "page-sub-header-for-profiler-configuration": "열 데이터 유형에 기반한 계산 메트릭을 설정하여 프로파일러의 동작을 전역적으로 사용자 정의하세요.", "page-sub-header-for-roles": "사용자 또는 팀에 포괄적인 역할 기반 접근 권한을 할당하세요.", "page-sub-header-for-search": "가장 인기 있는 검색 서비스에서 메타데이터를 수집하세요.", + "page-sub-header-for-search-index-mappings": "View and edit the Elasticsearch/OpenSearch index mappings for each entity type and language.", "page-sub-header-for-search-setting": "필요에 맞게 검색 설정을 구성할 수 있습니다.", "page-sub-header-for-security": "가장 인기 있는 보안 서비스에서 메타데이터를 수집하세요.", "page-sub-header-for-setting": "필요에 맞게 {{brandName}} 애플리케이션을 구성할 수 있습니다.", @@ -3690,6 +3695,7 @@ "search-for-edge": "파이프라인, 저장 프로시저 검색", "search-for-entity-types": "테이블, 토픽, 대시보드, 파이프라인, ML 모델, 용어집 및 태그 검색.", "search-for-ingestion": "수집 검색", + "search-index-mappings-reindex-info": "Changes to a mapping are saved to settings and applied on the next reindex of that entity. Run a reindex after saving for changes to take effect.", "search-insights-description": "검색 클러스터 통계, 인덱스 세부정보를 확인하고 고아 인덱스를 관리합니다.", "search-settings-description": "각 엔티티별 랭킹 기반 검색 설정입니다.", "search-settings-for-entity": "{{entity}}의 랭킹 기반 검색 설정", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/mr-in.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/mr-in.json index 8abc79fd44d8..f8996605b4e8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/mr-in.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/mr-in.json @@ -1988,6 +1988,7 @@ "reset-default-layout": "मूलभूत लेआउट रीसेट करा", "reset-entity": "{{entity}} रीसेट करा", "reset-position": "मध्यभागी रीसेट करा", + "reset-to-default": "Reset to default", "reset-view": "दृश्य रीसेट करा", "reset-your-password": "तुमचा पासवर्ड रीसेट करा", "resolution": "ठराव", @@ -2120,6 +2121,8 @@ "search-index-plural": "शोध अनुक्रमणिका", "search-index-setting-plural": "शोध अनुक्रमणिका सेटिंग्ज", "search-insights": "शोध अंतर्दृष्टी", + "search-mapping": "Search Mapping", + "search-mapping-plural": "Search Mappings", "search-memories": "सामग्रीद्वारे स्मृती शोधा...", "search-rbac": "RBAC शोधा", "search-result-plural": "शोध परिणाम", @@ -3266,6 +3269,7 @@ "invalid-dimensions": "अवैध प्रतिमा परिमाणे", "invalid-file": "अवैध फाइल", "invalid-file-format": "अवैध फाइल स्वरूप. स्वीकृत स्वरूपे: {{formats}}", + "invalid-json-mapping": "Invalid JSON. Please fix the mapping before saving.", "invalid-object-key": "अवैध ऑब्जेक्ट की. अक्षर, अंडरस्कोर किंवा डॉलर चिन्हाने सुरू होणे आवश्यक आहे, त्यानंतर अक्षरे, अंडरस्कोर्स, डॉलर चिन्हे किंवा अंक.", "invalid-odcs-contract-format": "अवैध ODCS करार स्वरूप. आवश्यक फील्ड: apiVersion, kind, status.", "invalid-odcs-contract-format-required-fields": "अगदी आवश्यक फील्ड: APIVersion, Kind, Status", @@ -3580,6 +3584,7 @@ "page-sub-header-for-profiler-configuration": "स्तंभ डेटा प्रकारांवर आधारित मेट्रिक्स सेट करून प्रोफाइलरचे वर्तन जागतिक स्तरावर सानुकूलित करा", "page-sub-header-for-roles": "वापरकर्ते किंवा टीम्सना व्यापक भूमिका आधारित प्रवेश नियुक्त करा.", "page-sub-header-for-search": "सर्वात लोकप्रिय शोध सेवांमधून मेटाडेटा अंतर्ग्रहण करा.", + "page-sub-header-for-search-index-mappings": "View and edit the Elasticsearch/OpenSearch index mappings for each entity type and language.", "page-sub-header-for-search-setting": "तुमच्या गरजा पूर्ण करण्यासाठी शोध सेटिंग्ज कॉन्फिगर करण्याची क्षमता.", "page-sub-header-for-security": "सर्वात लोकप्रिय सुरक्षा सेवांमधून मेटाडेटा अंतर्ग्रहण करा.", "page-sub-header-for-setting": "तुमच्या गरजा पूर्ण करण्यासाठी {{brandName}} अनुप्रयोग कॉन्फिगर करण्याची क्षमता.", @@ -3690,6 +3695,7 @@ "search-for-edge": "पाइपलाइन, StoredProcedures साठी शोधा", "search-for-entity-types": "टेबल्स, विषय, डॅशबोर्ड, पाइपलाइन, ML मॉडेल्स, शब्दकोश आणि टॅग्स साठी शोधा.", "search-for-ingestion": "अंतर्ग्रहण साठी शोधा", + "search-index-mappings-reindex-info": "Changes to a mapping are saved to settings and applied on the next reindex of that entity. Run a reindex after saving for changes to take effect.", "search-insights-description": "शोध क्लस्टर आकडेवारी, निर्देशांक तपशील पहा आणि अनाथ निर्देशांक व्यवस्थापित करा.", "search-settings-description": "रँकिंगवर आधारित प्रत्येक एंटिटीसाठी शोध सेटिंग्ज.", "search-settings-for-entity": "{{entity}} साठी रँकिंगवर आधारित शोध सेटिंग्ज", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json index db57e653441e..eb59c97485f3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json @@ -1988,6 +1988,7 @@ "reset-default-layout": "Standaardindeling herstellen", "reset-entity": "{{entity}} herstellen", "reset-position": "Resetten naar midden", + "reset-to-default": "Reset to default", "reset-view": "Weergave herstellen", "reset-your-password": "Herstel je wachtwoord", "resolution": "Resolutie", @@ -2120,6 +2121,8 @@ "search-index-plural": "Zoekindexen", "search-index-setting-plural": "Instellingen voor zoekindexen", "search-insights": "Zoekinzichten", + "search-mapping": "Search Mapping", + "search-mapping-plural": "Search Mappings", "search-memories": "Zoek herinneringen op inhoud...", "search-rbac": "Zoek RBAC", "search-result-plural": "Zoekresultaten", @@ -3266,6 +3269,7 @@ "invalid-dimensions": "Ongeldige afbeeldingsafmetingen", "invalid-file": "Ongeldig bestand", "invalid-file-format": "Ongeldig bestandsformaat. Geaccepteerde formaten: {{formats}}", + "invalid-json-mapping": "Invalid JSON. Please fix the mapping before saving.", "invalid-object-key": "Ongeldige objectnaam. Moet beginnen met een letter, underscore of dollarteken, gevolgd door letters, underscores, dollartekens of cijfers.", "invalid-odcs-contract-format": "Ongeldig ODCS-contractformaat. Vereiste velden: apiVersion, kind, status.", "invalid-odcs-contract-format-required-fields": "De volgende velden zijn vereist: APIVersion, Kind, Status", @@ -3580,6 +3584,7 @@ "page-sub-header-for-profiler-configuration": "Pas het gedrag van de profiler globaal aan door de metrische gegevens zo in te stellen dat ze worden berekend op basis van kolomgegevenstypen", "page-sub-header-for-roles": "Wijs uitgebreide rolgebaseerde toegang toe aan gebruikers of teams.", "page-sub-header-for-search": "Ingest metadata van de meestgebruikte zoekservices.", + "page-sub-header-for-search-index-mappings": "View and edit the Elasticsearch/OpenSearch index mappings for each entity type and language.", "page-sub-header-for-search-setting": "Mogelijkheid om de zoekinstellingen aan te passen aan uw behoeften.", "page-sub-header-for-security": "Ingest metadata van de meestgebruikte beveiligingsservices.", "page-sub-header-for-setting": "Mogelijkheid om de {{brandName}}-toepassing naar eigen behoefte te configureren.", @@ -3690,6 +3695,7 @@ "search-for-edge": "Zoeken naar Pipeline, Stored Procedures", "search-for-entity-types": "Zoeken naar Tabellen, Onderwerpen, Dashboards, Pipelines, ML-modellen, Woordenlijst en Tags.", "search-for-ingestion": "Zoeken naar ingestie", + "search-index-mappings-reindex-info": "Changes to a mapping are saved to settings and applied on the next reindex of that entity. Run a reindex after saving for changes to take effect.", "search-insights-description": "Bekijk zoekclusterstatistieken, indexdetails en beheer weesindexen.", "search-settings-description": "Zoekinstellingen voor elke entiteit gebaseerd op rankings.", "search-settings-for-entity": "Zoekinstellingen gebaseerd op ranking voor {{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pr-pr.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pr-pr.json index 999ca2ffbbe0..c540779f1e0b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pr-pr.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pr-pr.json @@ -1988,6 +1988,7 @@ "reset-default-layout": "بازنشانی طرح‌بندی پیش‌فرض", "reset-entity": "بازنشانی {{entity}}", "reset-position": "بازنشانی به مرکز", + "reset-to-default": "Reset to default", "reset-view": "بازنشانی نما", "reset-your-password": "بازنشانی رمز عبور خود", "resolution": "دقت", @@ -2120,6 +2121,8 @@ "search-index-plural": "شاخص‌های جستجو", "search-index-setting-plural": "تنظیمات شاخص جستجو", "search-insights": "بینش‌های جستجو", + "search-mapping": "Search Mapping", + "search-mapping-plural": "Search Mappings", "search-memories": "جستجوی خاطرات با محتوا...", "search-rbac": "Search RBAC", "search-result-plural": "نتایج جستجو", @@ -3266,6 +3269,7 @@ "invalid-dimensions": "ابعاد تصویر نامعتبر است", "invalid-file": "فایل نامعتبر", "invalid-file-format": "قالب فایل نامعتبر است. قالب‌های پذیرفته‌شده: {{formats}}", + "invalid-json-mapping": "Invalid JSON. Please fix the mapping before saving.", "invalid-object-key": "کلید شیء نامعتبر است. باید با یک حرف، زیرخط یا علامت دلار شروع شود و پس از آن حروف، زیرخط‌ها، علامت‌های دلار یا اعداد قرار گیرد.", "invalid-odcs-contract-format": "قالب قرارداد ODCS نامعتبر است. فیلدهای الزامی: apiVersion، kind، status.", "invalid-odcs-contract-format-required-fields": "الحقول المطلوبة: APIVersion, Kind, Status", @@ -3580,6 +3584,7 @@ "page-sub-header-for-profiler-configuration": "تنظیم رفتار پروفایلر در سطح جهانی با تعیین معیارها بر اساس نوع داده‌های ستون‌ها.", "page-sub-header-for-roles": "تخصیص دسترسی جامع مبتنی بر نقش به کاربران یا تیم‌ها.", "page-sub-header-for-search": "ورود متادیتا از پرکاربردترین سرویس‌های جستجو.", + "page-sub-header-for-search-index-mappings": "View and edit the Elasticsearch/OpenSearch index mappings for each entity type and language.", "page-sub-header-for-search-setting": "امکان پیکربندی تنظیمات جستجو متناسب با نیاز شما.", "page-sub-header-for-security": "ورود متادیتا از پرکاربردترین سرویس‌های امنیتی.", "page-sub-header-for-setting": "قابلیت پیکربندی برنامه {{brandName}} به‌طور دلخواه.", @@ -3690,6 +3695,7 @@ "search-for-edge": "جستجوی خطوط لوله، StoredProcedures.", "search-for-entity-types": "جستجوی جداول، موضوعات، داشبوردها، خطوط لوله، مدل‌های ML، واژه‌نامه و برچسب‌ها.", "search-for-ingestion": "جستجوی ورود داده.", + "search-index-mappings-reindex-info": "Changes to a mapping are saved to settings and applied on the next reindex of that entity. Run a reindex after saving for changes to take effect.", "search-insights-description": "آمار خوشه جستجو، جزئیات شاخص را مشاهده و شاخص‌های یتیم را مدیریت کنید.", "search-settings-description": "تنظیمات جستجو برای هر موجودیت بر اساس رتبه‌بندی.", "search-settings-for-entity": "تنظیمات جستجو بر اساس رتبه‌بندی برای {{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json index 22a165ff9a47..4b1465c472a2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json @@ -1988,6 +1988,7 @@ "reset-default-layout": "Redefinir Layout Padrão", "reset-entity": "Redefinir {{entity}}", "reset-position": "Redefinir para o centro", + "reset-to-default": "Reset to default", "reset-view": "Redefinir visualização", "reset-your-password": "Redefina sua Senha", "resolution": "Resolução", @@ -2120,6 +2121,8 @@ "search-index-plural": "Índices de Pesquisa", "search-index-setting-plural": "Configurações de Índice de Pesquisa", "search-insights": "Informações de pesquisa", + "search-mapping": "Search Mapping", + "search-mapping-plural": "Search Mappings", "search-memories": "Pesquisar memórias por conteúdo...", "search-rbac": "Pesquisar RBAC", "search-result-plural": "Resultados da pesquisa", @@ -3266,6 +3269,7 @@ "invalid-dimensions": "Dimensões de imagem inválidas", "invalid-file": "Arquivo inválido", "invalid-file-format": "Formato de arquivo inválido. Formatos aceitos: {{formats}}", + "invalid-json-mapping": "Invalid JSON. Please fix the mapping before saving.", "invalid-object-key": "Chave de objeto inválida. Deve começar com uma letra, sublinhado ou cifrão, seguido por letras, sublinhados, cifrões ou dígitos.", "invalid-odcs-contract-format": "Formato de contrato ODCS inválido. Campos obrigatórios: apiVersion, kind, status.", "invalid-odcs-contract-format-required-fields": "Os seguintes campos são obrigatórios: APIVersion, Kind, Status", @@ -3580,6 +3584,7 @@ "page-sub-header-for-profiler-configuration": "Personalize globalmente o comportamento do profiler configurando as métricas a serem calculadas com base nos tipos de dados das colunas.", "page-sub-header-for-roles": "Atribua acesso baseado em funções abrangentes a usuários ou equipes.", "page-sub-header-for-search": "Ingestão de metadados dos serviços de pesquisa mais populares.", + "page-sub-header-for-search-index-mappings": "View and edit the Elasticsearch/OpenSearch index mappings for each entity type and language.", "page-sub-header-for-search-setting": "Capacidade de definir as configurações de pesquisa para atender às suas necessidades.", "page-sub-header-for-security": "Ingestão de metadados dos serviços de segurança mais populares.", "page-sub-header-for-setting": "Habilidade para configurar a aplicação {{brandName}} de acordo com suas necessidades.", @@ -3690,6 +3695,7 @@ "search-for-edge": "Pesquisar por Pipeline, StoredProcedures", "search-for-entity-types": "Pesquisar por Tabelas, Tópicos, Painéis, Pipelines, Modelos de ML, Glossário e Tags.", "search-for-ingestion": "Pesquisar por ingestão", + "search-index-mappings-reindex-info": "Changes to a mapping are saved to settings and applied on the next reindex of that entity. Run a reindex after saving for changes to take effect.", "search-insights-description": "Visualize estatísticas de cluster de pesquisa, detalhes de índice e gerencie índices órfãos.", "search-settings-description": "Configurações de busca para cada entidade baseadas em rankings.", "search-settings-for-entity": "Configurações de busca baseadas em ranking para {{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-pt.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-pt.json index 666c72953560..048934a0828c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-pt.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-pt.json @@ -1988,6 +1988,7 @@ "reset-default-layout": "Redefinir Layout Padrão", "reset-entity": "Redefinir {{entity}}", "reset-position": "Redefinir para o centro", + "reset-to-default": "Reset to default", "reset-view": "Redefinir vista", "reset-your-password": "Redefina sua Senha", "resolution": "Resolução", @@ -2120,6 +2121,8 @@ "search-index-plural": "Índices de Pesquisa", "search-index-setting-plural": "Configurações de Índice de Pesquisa", "search-insights": "Insights de pesquisa", + "search-mapping": "Search Mapping", + "search-mapping-plural": "Search Mappings", "search-memories": "Pesquisar memórias por conteúdo...", "search-rbac": "Pesquisa RBAC", "search-result-plural": "Resultados da Pesquisa", @@ -3266,6 +3269,7 @@ "invalid-dimensions": "Dimensões de imagem inválidas", "invalid-file": "Arquivo inválido", "invalid-file-format": "Formato de arquivo inválido. Formatos aceitos: {{formats}}", + "invalid-json-mapping": "Invalid JSON. Please fix the mapping before saving.", "invalid-object-key": "Chave de objeto inválida. Deve começar com uma letra, sublinhado ou cifrão, seguido por letras, subliñados, cifrões ou dígitos.", "invalid-odcs-contract-format": "Formato de contrato ODCS inválido. Campos obrigatórios: apiVersion, kind, status.", "invalid-odcs-contract-format-required-fields": "Os seguintes campos são obrigatórios: APIVersion, Kind, Status", @@ -3580,6 +3584,7 @@ "page-sub-header-for-profiler-configuration": "Personalize globalmente o comportamento do profiler definindo as métricas a calcular com base nos tipos de dados das colunas", "page-sub-header-for-roles": "Atribua acesso baseado em funções abrangentes a Utilizadores ou equipas.", "page-sub-header-for-search": "Ingestão de metadados dos serviços de pesquisa mais populares.", + "page-sub-header-for-search-index-mappings": "View and edit the Elasticsearch/OpenSearch index mappings for each entity type and language.", "page-sub-header-for-search-setting": "Capacidade de configurar as definições de pesquisa para se adequar às suas necessidades.", "page-sub-header-for-security": "Ingestão de metadados dos serviços de segurança mais populares.", "page-sub-header-for-setting": "Habilidade para configurar a aplicação {{brandName}} de acordo com suas necessidades.", @@ -3690,6 +3695,7 @@ "search-for-edge": "Pesquisar por Pipeline, Procedimentos Armazenados", "search-for-entity-types": "Pesquisar por Tabelas, Tópicos, Painéis, Pipelines, Modelos de ML, Glossário e Etiquetas.", "search-for-ingestion": "Pesquisar por ingestão", + "search-index-mappings-reindex-info": "Changes to a mapping are saved to settings and applied on the next reindex of that entity. Run a reindex after saving for changes to take effect.", "search-insights-description": "Veja as estatísticas do cluster de pesquisa, os detalhes do índice e faça a gestão dos índices órfãos.", "search-settings-description": "Definições de pesquisa para cada entidade baseadas em classificações.", "search-settings-for-entity": "Definições de pesquisa baseadas em classificação para {{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json index ca1ea1f06971..ccae16acc2ec 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json @@ -1988,6 +1988,7 @@ "reset-default-layout": "Сбросить стандартный макет", "reset-entity": "Сбросить {{entity}}", "reset-position": "Сбросить в центр", + "reset-to-default": "Reset to default", "reset-view": "Сбросить представление", "reset-your-password": "Сбросить пароль", "resolution": "Разрешение", @@ -2120,6 +2121,8 @@ "search-index-plural": "Индексы поиска", "search-index-setting-plural": "Настройки индексов поиска", "search-insights": "Поисковая статистика", + "search-mapping": "Search Mapping", + "search-mapping-plural": "Search Mappings", "search-memories": "Поиск воспоминаний по содержимому...", "search-rbac": "Поиск RBAC", "search-result-plural": "Результаты поиска", @@ -3266,6 +3269,7 @@ "invalid-dimensions": "Недопустимые размеры изображения", "invalid-file": "Недопустимый файл", "invalid-file-format": "Недопустимый формат файла. Допустимые форматы: {{formats}}", + "invalid-json-mapping": "Invalid JSON. Please fix the mapping before saving.", "invalid-object-key": "Неверный ключ объекта. Он должен начинаться с буквы, знака подчеркивания или знака доллара, за которыми следуют буквы, знаки подчеркивания, знаки доллара или цифры.", "invalid-odcs-contract-format": "Недопустимый формат контракта ODCS. Обязательные поля: apiVersion, kind, status.", "invalid-odcs-contract-format-required-fields": "Обязательные поля: APIVersion, Kind, Status", @@ -3580,6 +3584,7 @@ "page-sub-header-for-profiler-configuration": "Настройте глобально поведение профилировщика, установив метрики для вычисления на основе типов данных столбцов", "page-sub-header-for-roles": "Назначьте полный доступ на основе ролей пользователям или командам.", "page-sub-header-for-search": "Принимайте метаданные из самых популярных поисковых сервисов.", + "page-sub-header-for-search-index-mappings": "View and edit the Elasticsearch/OpenSearch index mappings for each entity type and language.", "page-sub-header-for-search-setting": "Возможность настроить поиск", "page-sub-header-for-security": "Получение метаданных из самых популярных служб безопасности.", "page-sub-header-for-setting": "Возможность настройки приложения {{brandName}} в соответствии с вашими потребностями.", @@ -3690,6 +3695,7 @@ "search-for-edge": "Поиск по пайплайнам, хранимым процедурам", "search-for-entity-types": "Поиск по таблицам, топикам, дашбордам, пайплайнам, моделям машинного обучения, глоссарию и тегам.", "search-for-ingestion": "Поиск для получения", + "search-index-mappings-reindex-info": "Changes to a mapping are saved to settings and applied on the next reindex of that entity. Run a reindex after saving for changes to take effect.", "search-insights-description": "Просматривайте статистику поискового кластера, сведения об индексах и управляйте потерянными индексами.", "search-settings-description": "Настройки поиска для каждой сущности на основе рейтингов.", "search-settings-for-entity": "Настройки поиска на основе рейтинга для объекта «{{entity}}»", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/sv-se.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/sv-se.json index c9dcbd42a6e2..b08157539679 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/sv-se.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/sv-se.json @@ -1988,6 +1988,7 @@ "reset-default-layout": "Återställ standardlayout", "reset-entity": "Återställ {{entity}}", "reset-position": "Återställ till mitten", + "reset-to-default": "Reset to default", "reset-view": "Återställ vy", "reset-your-password": "Återställ ditt lösenord", "resolution": "Upplösning", @@ -2120,6 +2121,8 @@ "search-index-plural": "Sökindex", "search-index-setting-plural": "Inställningar för sökindex", "search-insights": "Sökinsikter", + "search-mapping": "Search Mapping", + "search-mapping-plural": "Search Mappings", "search-memories": "Sök minnen efter innehåll...", "search-rbac": "Sök RBAC", "search-result-plural": "Sökresultat", @@ -3266,6 +3269,7 @@ "invalid-dimensions": "Ogiltiga bilddimensioner", "invalid-file": "Ogiltig fil", "invalid-file-format": "Ogiltigt filformat. Godkända format: {{formats}}", + "invalid-json-mapping": "Invalid JSON. Please fix the mapping before saving.", "invalid-object-key": "Ogiltig objektnyckel. Måste börja med en bokstav, ett understreck eller ett dollartecken, följt av bokstäver, understreck, dollartecken eller siffror.", "invalid-odcs-contract-format": "Ogiltigt ODCS-kontraktsformat. Obligatoriska fält: apiVersion, kind, status.", "invalid-odcs-contract-format-required-fields": "Följande fält krävs: APIVersion, Kind, Status", @@ -3580,6 +3584,7 @@ "page-sub-header-for-profiler-configuration": "Anpassa globalt profilerarens beteende genom att ställa in vilka mätvärden som ska beräknas baserat på kolumners datatyper", "page-sub-header-for-roles": "Tilldela omfattande rollbaserad åtkomst till användare eller team.", "page-sub-header-for-search": "Mata in metadata från de mest populära söktjänsterna.", + "page-sub-header-for-search-index-mappings": "View and edit the Elasticsearch/OpenSearch index mappings for each entity type and language.", "page-sub-header-for-search-setting": "Möjlighet att konfigurera sökinställningarna för att passa dina behov.", "page-sub-header-for-security": "Mata in metadata från de mest populära säkerhetstjänsterna.", "page-sub-header-for-setting": "Möjlighet att konfigurera {{brandName}}-applikationen för att passa dina behov.", @@ -3690,6 +3695,7 @@ "search-for-edge": "Sök efter Pipeline, StoredProcedures", "search-for-entity-types": "Sök efter tabeller, ämnen, instrumentpaneler, pipelines, ML-modeller, ordlista och taggar.", "search-for-ingestion": "Sök efter inmatning", + "search-index-mappings-reindex-info": "Changes to a mapping are saved to settings and applied on the next reindex of that entity. Run a reindex after saving for changes to take effect.", "search-insights-description": "Visa statistik för sökkluster, indexdetaljer och hantera föräldralösa index.", "search-settings-description": "Sökinställningar för varje entitet baserat på rangordningar.", "search-settings-for-entity": "Sök inställningar baserat på rankning för {{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/th-th.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/th-th.json index ec5e09c0ce4b..1d1e0aacc282 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/th-th.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/th-th.json @@ -1988,6 +1988,7 @@ "reset-default-layout": "รีเซ็ตเลย์เอาต์เริ่มต้น", "reset-entity": "รีเซ็ต{{entity}}", "reset-position": "รีเซ็ตไปที่ตรงกลาง", + "reset-to-default": "Reset to default", "reset-view": "รีเซ็ตมุมมอง", "reset-your-password": "รีเซ็ตรหัสผ่านของคุณ", "resolution": "การแก้ไข", @@ -2120,6 +2121,8 @@ "search-index-plural": "ดัชนีการค้นหาหลายรายการ", "search-index-setting-plural": "การตั้งค่าดัชนีการค้นหา", "search-insights": "ข้อมูลเชิงลึกการค้นหา", + "search-mapping": "Search Mapping", + "search-mapping-plural": "Search Mappings", "search-memories": "ค้นหาความทรงจำตามเนื้อหา...", "search-rbac": "ค้นหา RBAC", "search-result-plural": "ผลการค้นหา", @@ -3266,6 +3269,7 @@ "invalid-dimensions": "ขนาดภาพไม่ถูกต้อง", "invalid-file": "ไฟล์ไม่ถูกต้อง", "invalid-file-format": "รูปแบบไฟล์ไม่ถูกต้อง รูปแบบที่ยอมรับ: {{formats}}", + "invalid-json-mapping": "Invalid JSON. Please fix the mapping before saving.", "invalid-object-key": "คีย์วัตถุไม่ถูกต้อง ต้องเริ่มต้นด้วยตัวอักษร, ขีดล่าง, หรือเครื่องหมายดอลลาร์ ตามด้วยตัวอักษร, ขีดล่าง, เครื่องหมายดอลลาร์, หรือหลักฐาน", "invalid-odcs-contract-format": "รูปแบบสัญญา ODCS ไม่ถูกต้อง ฟิลด์ที่ต้องการ: apiVersion, kind, status", "invalid-odcs-contract-format-required-fields": "ฟิลด์ที่ต้องการ: APIVersion, Kind, Status", @@ -3580,6 +3584,7 @@ "page-sub-header-for-profiler-configuration": "ปรับแต่งพฤติกรรมของโปรไฟล์เลอร์ในระดับโลกโดยการตั้งค่าเมตริกที่จะคำนวณตามประเภทของข้อมูลในคอลัมน์", "page-sub-header-for-roles": "มอบบทบาทการเข้าถึงที่ครอบคลุมให้กับผู้ใช้หรือทีม", "page-sub-header-for-search": "นำเข้าข้อมูลเมตาจากบริการค้นหาที่ได้รับความนิยมมากที่สุด", + "page-sub-header-for-search-index-mappings": "View and edit the Elasticsearch/OpenSearch index mappings for each entity type and language.", "page-sub-header-for-search-setting": "ความสามารถในการกำหนดค่าการตั้งค่าการค้นหาเพื่อตอบสนองความต้องการของคุณ", "page-sub-header-for-security": "นำเข้าข้อมูลเมตาจากบริการค้นหาที่ได้รับความนิยมมากที่สุด", "page-sub-header-for-setting": "ความสามารถในการกำหนดค่าแอปพลิเคชัน {{brandName}} เพื่อตอบสนองความต้องการของคุณ", @@ -3690,6 +3695,7 @@ "search-for-edge": "ค้นหาท่อ, กระบวนการที่เก็บ", "search-for-entity-types": "ค้นหาตาราง, หัวข้อ, แดชบอร์ด, ท่อ, โมเดล ML, สารานุกรม และแท็ก", "search-for-ingestion": "ค้นหาการนำเข้า", + "search-index-mappings-reindex-info": "Changes to a mapping are saved to settings and applied on the next reindex of that entity. Run a reindex after saving for changes to take effect.", "search-insights-description": "ดูสถิติคลัสเตอร์การค้นหา รายละเอียดดัชนี และจัดการดัชนีกำพร้า", "search-settings-description": "การตั้งค่าการค้นหาสำหรับแต่ละเอนทิตีตามการจัดอันดับ", "search-settings-for-entity": "การตั้งค่าการค้นหาตามการจัดอันดับสำหรับ {{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/tr-tr.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/tr-tr.json index d94d2a1e51ed..66316c5f5918 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/tr-tr.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/tr-tr.json @@ -1988,6 +1988,7 @@ "reset-default-layout": "Varsayılan Düzeni Sıfırla", "reset-entity": "{{entity}} Sıfırla", "reset-position": "Merkeze sıfırla", + "reset-to-default": "Reset to default", "reset-view": "Görünümü sıfırla", "reset-your-password": "Şifrenizi Sıfırlayın", "resolution": "Çözünürlük", @@ -2120,6 +2121,8 @@ "search-index-plural": "Arama Dizinleri", "search-index-setting-plural": "Arama Dizini Ayarları", "search-insights": "Arama İçgörüleri", + "search-mapping": "Search Mapping", + "search-mapping-plural": "Search Mappings", "search-memories": "Anıları içeriğe göre ara...", "search-rbac": "RBAC Ara", "search-result-plural": "Arama Sonuçları", @@ -3266,6 +3269,7 @@ "invalid-dimensions": "Geçersiz görsel boyutları", "invalid-file": "Geçersiz dosya", "invalid-file-format": "Geçersiz dosya formatı. Kabul edilen formatlar: {{formats}}", + "invalid-json-mapping": "Invalid JSON. Please fix the mapping before saving.", "invalid-object-key": "Geçersiz nesne anahtarı. Bir harf, alt çizgi veya dolar işareti ile başlamalı, ardından harfler, alt çizgiler, dolar işaretleri veya rakamlar gelmelidir.", "invalid-odcs-contract-format": "Geçersiz ODCS sözleşme formatı. Gerekli alanlar: apiVersion, kind, status.", "invalid-odcs-contract-format-required-fields": "Gerekli alanlar: APIVersion, Kind, Status", @@ -3580,6 +3584,7 @@ "page-sub-header-for-profiler-configuration": "Sütun veri türlerine göre hesaplanacak metrikleri ayarlayarak profilleyicinin davranışını genel olarak özelleştirin", "page-sub-header-for-roles": "Kullanıcılara veya Takımlara kapsamlı rol tabanlı erişim atayın.", "page-sub-header-for-search": "En popüler arama servislerinden metadata alın.", + "page-sub-header-for-search-index-mappings": "View and edit the Elasticsearch/OpenSearch index mappings for each entity type and language.", "page-sub-header-for-search-setting": "İhtiyaçlarınıza uygun arama ayarlarını yapılandırma yeteneği.", "page-sub-header-for-security": "En popüler güvenlik servislerinden metadata alın.", "page-sub-header-for-setting": "{{brandName}} uygulamasını ihtiyaçlarınıza göre yapılandırma yeteneği.", @@ -3690,6 +3695,7 @@ "search-for-edge": "İş Akışı, Saklı Yordamlar için Ara", "search-for-entity-types": "Tablolar, Konular, Kontrol Panelleri, İş Akışları, ML Modelleri, Sözlük ve Etiketler için arama yapın.", "search-for-ingestion": "Alım için ara", + "search-index-mappings-reindex-info": "Changes to a mapping are saved to settings and applied on the next reindex of that entity. Run a reindex after saving for changes to take effect.", "search-insights-description": "Arama kümesi istatistiklerini, dizin ayrıntılarını görüntüleyin ve yetim dizinleri yönetin.", "search-settings-description": "Sıralamaya göre her varlık için arama ayarları.", "search-settings-for-entity": "{{entity}} için sıralamaya dayalı arama ayarları", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json index 2a369550ca9f..686c4b63c835 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json @@ -1988,6 +1988,7 @@ "reset-default-layout": "重置默认布局", "reset-entity": "重置{{entity}}", "reset-position": "重置到中心", + "reset-to-default": "Reset to default", "reset-view": "重置视图", "reset-your-password": "重置密码", "resolution": "解决方案", @@ -2120,6 +2121,8 @@ "search-index-plural": "搜索索引", "search-index-setting-plural": "搜索索引设置", "search-insights": "搜索见解", + "search-mapping": "Search Mapping", + "search-mapping-plural": "Search Mappings", "search-memories": "按内容搜索记忆...", "search-rbac": "搜索 RBAC", "search-result-plural": "搜索结果", @@ -3266,6 +3269,7 @@ "invalid-dimensions": "无效的图片尺寸", "invalid-file": "无效的文件", "invalid-file-format": "无效的文件格式。支持的格式：{{formats}}", + "invalid-json-mapping": "Invalid JSON. Please fix the mapping before saving.", "invalid-object-key": "无效的对象键名, 命名首字母必须由字母、下划线、美元符号开始, 再跟随字母、下划线、美元符号或数字", "invalid-odcs-contract-format": "无效的 ODCS 合同格式。必填字段：apiVersion、kind、status。", "invalid-odcs-contract-format-required-fields": "必填字段：APIVersion, Kind, Status", @@ -3580,6 +3584,7 @@ "page-sub-header-for-profiler-configuration": "根据列数据类型计算的指标, 设置全局自定义分析器的行为", "page-sub-header-for-roles": "分配基于角色的访问权限给用户或团队", "page-sub-header-for-search": "从最流行的搜索服务中提取元数据", + "page-sub-header-for-search-index-mappings": "View and edit the Elasticsearch/OpenSearch index mappings for each entity type and language.", "page-sub-header-for-search-setting": "能够配置搜索设置以满足您的需求。", "page-sub-header-for-security": "从最流行的安全服务中提取元数据", "page-sub-header-for-setting": "能够根据需要配置 {{brandName}} 应用", @@ -3690,6 +3695,7 @@ "search-for-edge": "搜索工作流、存储过程", "search-for-entity-types": "搜索数据表、消息主题、仪表板、工作流和机械学习模型", "search-for-ingestion": "搜索提取作业", + "search-index-mappings-reindex-info": "Changes to a mapping are saved to settings and applied on the next reindex of that entity. Run a reindex after saving for changes to take effect.", "search-insights-description": "查看搜索集群统计信息、索引详细信息并管理孤立索引。", "search-settings-description": "基于排名的每个实体的搜索设置。", "search-settings-for-entity": "基于排名的{{entity}}搜索设置", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-tw.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-tw.json index d791454af503..cdc8f7aea937 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-tw.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-tw.json @@ -1988,6 +1988,7 @@ "reset-default-layout": "重設預設版面配置", "reset-entity": "重設{{entity}}", "reset-position": "重置到中心", + "reset-to-default": "Reset to default", "reset-view": "重設檢視", "reset-your-password": "重設您的密碼", "resolution": "解決方案", @@ -2120,6 +2121,8 @@ "search-index-plural": "搜尋索引", "search-index-setting-plural": "搜尋索引設定", "search-insights": "搜尋洞察", + "search-mapping": "Search Mapping", + "search-mapping-plural": "Search Mappings", "search-memories": "按內容搜索記憶...", "search-rbac": "搜尋 RBAC", "search-result-plural": "搜尋結果", @@ -3266,6 +3269,7 @@ "invalid-dimensions": "無效的圖片尺寸", "invalid-file": "無效的文件", "invalid-file-format": "無效的文件格式。支持的格式：{{formats}}", + "invalid-json-mapping": "Invalid JSON. Please fix the mapping before saving.", "invalid-object-key": "無效的物件金鑰。必須以字母、底線或貨幣符號開頭，後面跟著字母、底線、貨幣符號或數字。", "invalid-odcs-contract-format": "無效的 ODCS 合約格式。必填欄位：apiVersion、kind、status。", "invalid-odcs-contract-format-required-fields": "必填欄位：APIVersion, Kind, Status", @@ -3580,6 +3584,7 @@ "page-sub-header-for-profiler-configuration": "透過設定基於欄位資料類型的計算指標，全域自訂分析器的行為", "page-sub-header-for-roles": "將全面的基於角色的存取權限指派給使用者或團隊。", "page-sub-header-for-search": "從最受歡迎的搜尋服務擷取元資料。", + "page-sub-header-for-search-index-mappings": "View and edit the Elasticsearch/OpenSearch index mappings for each entity type and language.", "page-sub-header-for-search-setting": "能夠設定搜尋設定以滿足您的需求。", "page-sub-header-for-security": "從最受歡迎的安全服務中擷取元資料。", "page-sub-header-for-setting": "能夠設定 {{brandName}} 應用程式以滿足您的需求。", @@ -3690,6 +3695,7 @@ "search-for-edge": "搜尋管線、預存程序", "search-for-entity-types": "搜尋資料表、主題、儀表板、管線、機器學習模型、詞彙表和標籤。", "search-for-ingestion": "搜尋擷取", + "search-index-mappings-reindex-info": "Changes to a mapping are saved to settings and applied on the next reindex of that entity. Run a reindex after saving for changes to take effect.", "search-insights-description": "檢視搜尋叢集統計、索引詳細資料並管理孤立索引。", "search-settings-description": "根據排名對每個實體進行搜尋設定。", "search-settings-for-entity": "根據 {{entity}} 的排名進行搜尋設定", diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/SearchIndexMappingsPage/SearchIndexMappingsPage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/SearchIndexMappingsPage/SearchIndexMappingsPage.component.tsx new file mode 100644 index 000000000000..01769678a91f --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/pages/SearchIndexMappingsPage/SearchIndexMappingsPage.component.tsx @@ -0,0 +1,335 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + Alert, + Badge, + Button, + Card, + Select, + SelectItemType, +} from '@openmetadata/ui-core-components'; +import { AxiosError } from 'axios'; +import { isEmpty } from 'lodash'; +import { useEffect, useMemo, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import HeaderBreadcrumb from '../../components/common/HeaderBreadcrumb/HeaderBreadcrumb.component'; +import Loader from '../../components/common/Loader/Loader'; +import SchemaEditor from '../../components/Database/SchemaEditor/SchemaEditor'; +import PageHeader from '../../components/PageHeader/PageHeader.component'; +import PageLayoutV1 from '../../components/PageLayoutV1/PageLayoutV1'; +import { JSON_TAB_SIZE } from '../../constants/constants'; +import { GlobalSettingsMenuCategory } from '../../constants/GlobalSettings.constants'; +import { PAGE_HEADERS } from '../../constants/PageHeaders.constant'; +import { CSMode } from '../../enums/codemirror.enum'; +import { + getSearchIndexMapping, + getSearchIndexMappingsList, + resetSearchIndexMapping, + SearchIndexMapping, + SearchIndexMappingsList, + updateSearchIndexMapping, +} from '../../rest/settingConfigAPI'; +import { getSettingPageEntityBreadCrumb } from '../../utils/GlobalSettingsUtils'; +import { showErrorToast, showSuccessToast } from '../../utils/ToastUtils'; +import './search-index-mappings-page.less'; + +const DEFAULT_LANGUAGE = 'en'; + +/** + * Admin page to view and edit the per-language, per-entity Elasticsearch/ + * OpenSearch index mappings stored in settings. Saving only persists the + * mapping; the change applies on the next reindex of that entity. A reindex + * trigger is intentionally omitted here — the info banner instructs the admin + * to reindex after saving. + */ +const SearchIndexMappingsPage = () => { + const { t } = useTranslation(); + + const [isLoading, setIsLoading] = useState(true); + const [isMappingLoading, setIsMappingLoading] = useState(false); + const [isSaving, setIsSaving] = useState(false); + const [isResetting, setIsResetting] = useState(false); + const [mappingsList, setMappingsList] = useState({}); + const [selectedLanguage, setSelectedLanguage] = useState(''); + const [selectedEntityType, setSelectedEntityType] = useState(''); + const [editorValue, setEditorValue] = useState(''); + + const breadcrumbs = useMemo( + () => + getSettingPageEntityBreadCrumb( + GlobalSettingsMenuCategory.PREFERENCES, + t('label.search-mapping-plural') + ).map((link) => ({ href: link.url, label: link.name })), + [t] + ); + + const languageItems: SelectItemType[] = useMemo( + () => + Object.keys(mappingsList).map((language) => ({ + id: language, + label: language.toUpperCase(), + })), + [mappingsList] + ); + + const entityTypeItems: SelectItemType[] = useMemo( + () => + (mappingsList[selectedLanguage] ?? []).map((entityType) => ({ + id: entityType, + label: entityType, + })), + [mappingsList, selectedLanguage] + ); + + const hasMappings = !isEmpty(mappingsList); + const isEntitySelected = + !isEmpty(selectedLanguage) && !isEmpty(selectedEntityType); + const isActionDisabled = isMappingLoading || !isEntitySelected; + + const fetchMappingsList = async () => { + try { + setIsLoading(true); + const data = await getSearchIndexMappingsList(); + setMappingsList(data); + + const languages = Object.keys(data); + const defaultLanguage = languages.includes(DEFAULT_LANGUAGE) + ? DEFAULT_LANGUAGE + : languages[0] ?? ''; + setSelectedLanguage(defaultLanguage); + setSelectedEntityType(data[defaultLanguage]?.[0] ?? ''); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setIsLoading(false); + } + }; + + const fetchMapping = async (language: string, entityType: string) => { + try { + setIsMappingLoading(true); + const mapping = await getSearchIndexMapping(language, entityType); + setEditorValue(JSON.stringify(mapping, null, JSON_TAB_SIZE)); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setIsMappingLoading(false); + } + }; + + const handleLanguageChange = (language: string) => { + setSelectedLanguage(language); + setSelectedEntityType(mappingsList[language]?.[0] ?? ''); + }; + + const handleFormat = () => { + try { + setEditorValue( + JSON.stringify(JSON.parse(editorValue), null, JSON_TAB_SIZE) + ); + } catch (error) { + showErrorToast(t('message.invalid-json-mapping')); + } + }; + + const handleSave = async () => { + let parsedMapping: SearchIndexMapping | undefined; + try { + parsedMapping = JSON.parse(editorValue) as SearchIndexMapping; + } catch (error) { + showErrorToast(t('message.invalid-json-mapping')); + } + + if (parsedMapping) { + try { + setIsSaving(true); + await updateSearchIndexMapping( + selectedLanguage, + selectedEntityType, + parsedMapping + ); + showSuccessToast( + t('server.update-entity-success', { + entity: t('label.search-mapping-plural'), + }) + ); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setIsSaving(false); + } + } + }; + + const handleReset = async () => { + try { + setIsResetting(true); + await resetSearchIndexMapping(selectedLanguage, selectedEntityType); + await fetchMapping(selectedLanguage, selectedEntityType); + showSuccessToast( + t('server.update-entity-success', { + entity: t('label.search-mapping-plural'), + }) + ); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setIsResetting(false); + } + }; + + useEffect(() => { + fetchMappingsList(); + }, []); + + useEffect(() => { + if (isEntitySelected) { + fetchMapping(selectedLanguage, selectedEntityType); + } + }, [selectedLanguage, selectedEntityType]); + + if (isLoading) { + return ; + } + + return ( + +
+ + +
+ +
+ + +
+
+ + + + + +
+ + +
+ + + + + + {t('label.format')} + + } + title={ +
+ {selectedEntityType || '--'} + {selectedLanguage ? ( + + {selectedLanguage.toUpperCase()} + + ) : null} +
+ } + /> + + {!hasMappings || !isEntitySelected ? ( +
+ {t('message.no-data-available')} +
+ ) : isMappingLoading ? ( +
+ +
+ ) : ( + + )} + + +
+ + ); +}; + +export default SearchIndexMappingsPage; diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/SearchIndexMappingsPage/search-index-mappings-page.less b/openmetadata-ui/src/main/resources/ui/src/pages/SearchIndexMappingsPage/search-index-mappings-page.less new file mode 100644 index 000000000000..4e9d28dc248c --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/pages/SearchIndexMappingsPage/search-index-mappings-page.less @@ -0,0 +1,30 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@import (reference) '../../styles/variables.less'; + +.search-index-mappings-page { + padding: 16px; + + &__editor .CodeMirror { + height: calc(100vh - 380px); + min-height: 420px; + } + + &__editor-loader, + &__empty { + display: flex; + align-items: center; + justify-content: center; + min-height: 420px; + } +} diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/settingConfigAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/settingConfigAPI.ts index 02136706af17..81b43ff7c774 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/settingConfigAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/settingConfigAPI.ts @@ -139,6 +139,57 @@ export const getSystemConfig = async () => { return response.data; }; +export type SearchIndexMappingsList = Record; + +export type SearchIndexMapping = Record; + +export const getSearchIndexMappingsList = + async (): Promise => { + const response = await axiosClient.get( + '/system/settings/searchIndexMappings' + ); + + return response.data; + }; + +export const getSearchIndexMapping = async ( + language: string, + entityType: string, + fallback = true +): Promise => { + const response = await axiosClient.get( + `/system/settings/searchIndexMappings/${language}/${entityType}`, + { params: { fallback } } + ); + + return response.data; +}; + +export const updateSearchIndexMapping = async ( + language: string, + entityType: string, + mapping: SearchIndexMapping +): Promise => { + const response = await axiosClient.put( + `/system/settings/searchIndexMappings/${language}/${entityType}`, + mapping, + APPLICATION_JSON_CONTENT_TYPE_HEADER + ); + + return response.data; +}; + +export const resetSearchIndexMapping = async ( + language: string, + entityType: string +): Promise => { + const response = await axiosClient.put( + `/system/settings/searchIndexMappings/reset/${language}/${entityType}` + ); + + return response.data; +}; + export const getGlossaryTermRelationSettings = async (): Promise => { const response = await axiosClient.get( diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/GlobalSettingsClassBase.ts b/openmetadata-ui/src/main/resources/ui/src/utils/GlobalSettingsClassBase.ts index d02d872e78dc..10e929c78b23 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/GlobalSettingsClassBase.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/GlobalSettingsClassBase.ts @@ -649,6 +649,13 @@ class GlobalSettingsClassBase { }, ].sort((a, b) => a.label.localeCompare(b.label)), }, + { + label: t('label.search-mapping-plural'), + description: t('message.page-sub-header-for-search-index-mappings'), + isProtected: Boolean(isAdminUser), + key: `${GlobalSettingsMenuCategory.PREFERENCES}.${GlobalSettingOptions.SEARCH_MAPPINGS}`, + icon: PreferencesSearchIcon, + }, { label: t('label.ai'), description: t('message.page-sub-header-for-ai-setting'),