From 9cf6116382af8718198bbd871ea0655e3e309b36 Mon Sep 17 00:00:00 2001 From: guojialiang Date: Thu, 2 Apr 2026 20:51:36 +0800 Subject: [PATCH 1/2] When a query fails due to the number of search only shards being 0, provide clear exception information. Signed-off-by: guojialiang --- .../scale/searchonly/ScaleIndexIT.java | 33 +++++++++- .../indices/settings/SearchOnlyReplicaIT.java | 45 ++++++++++++++ .../cluster/routing/OperationRouting.java | 25 ++++++++ .../routing/OperationRoutingTests.java | 60 ++++++++++++++++++- .../ClusterStateCreationUtils.java | 6 +- 5 files changed, 162 insertions(+), 7 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/scale/searchonly/ScaleIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/scale/searchonly/ScaleIndexIT.java index 97d0f8565394d..551cbfa077aa7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/scale/searchonly/ScaleIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/scale/searchonly/ScaleIndexIT.java @@ -8,6 +8,7 @@ package org.opensearch.action.admin.indices.scale.searchonly; +import org.opensearch.action.NoShardAvailableActionException; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.SearchResponse; @@ -16,6 +17,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; +import org.opensearch.cluster.routing.Preference; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.settings.Settings; import org.opensearch.core.rest.RestStatus; @@ -25,6 +27,7 @@ import org.opensearch.test.OpenSearchIntegTestCase; import java.util.HashSet; +import java.util.Locale; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -88,9 +91,17 @@ private void testFullLifecycle(int searchOnlyReplica) throws Exception { assertSearchNodeDocCounts(10, TEST_INDEX); } else { try { - client().prepareSearch(TEST_INDEX).setSize(0).get(); - } catch (Exception e) { - assertTrue(e.getMessage().contains("all shards failed")); + client().prepareSearch(TEST_INDEX).setPreference(Preference.SEARCH_REPLICA.type()).setSize(0).get(); + fail("search replica should have failed"); + } catch (NoShardAvailableActionException e) { + assertEquals( + String.format( + Locale.ROOT, + "Strictly require querying search only shards, but the number of search only replicas for index %s is 0", + TEST_INDEX + ), + e.getMessage() + ); } } }, 30, TimeUnit.SECONDS); @@ -124,6 +135,22 @@ private void testFullLifecycle(int searchOnlyReplica) throws Exception { ensureGreen(TEST_INDEX); + if (searchOnlyReplica == 0) { + try { + client().prepareSearch(TEST_INDEX).setSize(0).get(); + fail("search replica should have failed"); + } catch (NoShardAvailableActionException e) { + assertEquals( + String.format( + Locale.ROOT, + "The index %s is in the scale down state, and the number of search only shards is 0", + TEST_INDEX + ), + e.getMessage() + ); + } + } + assertAcked( client().admin() .indices() diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaIT.java index db34bb113320d..58c4bee6ecf4f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaIT.java @@ -8,6 +8,7 @@ package org.opensearch.indices.settings; +import org.opensearch.action.NoShardAvailableActionException; import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.WriteRequest; @@ -25,6 +26,7 @@ import org.opensearch.test.OpenSearchIntegTestCase; import java.io.IOException; +import java.util.Locale; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; @@ -238,6 +240,49 @@ public void testSearchReplicaRoutingPreference() throws IOException { assertEquals(nodeId, indexShardRoutingTable.searchOnlyReplicas().get(0).currentNodeId()); } + public void testSearchReplicaRoutingPreferenceWithoutSearchReplicaShard() throws IOException { + int numSearchReplicas = 0; + int numWriterReplicas = 1; + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNode(); + createIndex( + TEST_INDEX, + Settings.builder() + .put(indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numWriterReplicas) + .put(IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS, numSearchReplicas) + .build() + ); + ensureYellow(TEST_INDEX); + client().prepareIndex(TEST_INDEX).setId("1").setSource("foo", "bar").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + // add 2 nodes for the replicas + internalCluster().startDataOnlyNode(); + internalCluster().startSearchOnlyNode(); + + ensureGreen(TEST_INDEX); + + assertActiveShardCounts(numSearchReplicas, numWriterReplicas); + + assertHitCount(client().prepareSearch(TEST_INDEX).setPreference(null).setSize(0).get(), 1); + + Throwable throwable = assertThrows( + NoShardAvailableActionException.class, + () -> client().prepareSearch(TEST_INDEX) + .setPreference(Preference.SEARCH_REPLICA.type()) + .setQuery(QueryBuilders.matchAllQuery()) + .get() + ); + + assertEquals( + String.format( + Locale.ROOT, + "Strictly require querying search only shards, but the number of search only replicas for index %s is 0", + TEST_INDEX + ), + throwable.getMessage() + ); + } + public void testSearchReplicaRoutingPreferenceWhenSearchReplicaUnassigned() { internalCluster().startClusterManagerOnlyNode(); internalCluster().startDataOnlyNode(); diff --git a/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java b/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java index 210e9828876df..4506bb0089d06 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java +++ b/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java @@ -33,6 +33,7 @@ package org.opensearch.cluster.routing; import org.apache.lucene.util.CollectionUtil; +import org.opensearch.action.NoShardAvailableActionException; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.VirtualShardRoutingHelper; @@ -59,6 +60,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -281,6 +283,29 @@ public GroupShardsIterator searchShards( } } + if (indexMetadataForShard.getNumberOfSearchOnlyReplicas() == 0 && (Preference.SEARCH_REPLICA.type().equals(preference))) { + throw new NoShardAvailableActionException( + null, + String.format( + Locale.ROOT, + "Strictly require querying search only shards, but the number of search only replicas for index %s is 0", + indexMetadataForShard.getIndex().getName() + ) + ); + } + + if (indexMetadataForShard.getNumberOfSearchOnlyReplicas() == 0 + && indexMetadataForShard.getSettings().getAsBoolean(IndexMetadata.INDEX_BLOCKS_SEARCH_ONLY_SETTING.getKey(), false)) { + throw new NoShardAvailableActionException( + null, + String.format( + Locale.ROOT, + "The index %s is in the scale down state, and the number of search only shards is 0", + indexMetadataForShard.getIndex().getName() + ) + ); + } + ShardIterator iterator = preferenceActiveShardIterator( shard, clusterState.nodes().getLocalNodeId(), diff --git a/server/src/test/java/org/opensearch/cluster/routing/OperationRoutingTests.java b/server/src/test/java/org/opensearch/cluster/routing/OperationRoutingTests.java index 1254c1a84e573..65dc494168e29 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/OperationRoutingTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/OperationRoutingTests.java @@ -32,6 +32,7 @@ package org.opensearch.cluster.routing; import org.opensearch.Version; +import org.opensearch.action.NoShardAvailableActionException; import org.opensearch.action.support.replication.ClusterStateCreationUtils; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; @@ -62,6 +63,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.TreeMap; @@ -672,6 +674,58 @@ public void testAdaptiveReplicaSelection() throws Exception { terminate(threadPool); } + public void testSearchExceptionWithoutSearchOnlyReplicas() throws Exception { + final int numIndices = 1; + final String[] indexNames = new String[numIndices]; + for (int i = 0; i < numIndices; i++) { + indexNames[i] = "test" + i; + } + ClusterState state = ClusterStateCreationUtils.stateWithAssignedPrimariesAndReplicas(indexNames, 1, 1); + OperationRouting opRouting = new OperationRouting( + Settings.EMPTY, + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); + opRouting.setUseAdaptiveReplicaSelection(true); + TestThreadPool threadPool = new TestThreadPool("testThatOnlyNodesSupportNodeIds"); + ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool); + ResponseCollectorService collector = new ResponseCollectorService(clusterService); + Map outstandingRequests = new HashMap<>(); + Throwable throwable = assertThrows( + NoShardAvailableActionException.class, + () -> opRouting.searchShards(state, indexNames, null, Preference.SEARCH_REPLICA.type(), collector, outstandingRequests, null) + ); + + assertEquals( + String.format( + Locale.ROOT, + "Strictly require querying search only shards, but the number of search only replicas for index %s is 0", + "test0" + ), + throwable.getMessage() + ); + + ClusterState newState = ClusterStateCreationUtils.stateWithAssignedPrimariesAndReplicas( + indexNames, + 1, + 1, + 0, + Settings.builder().put(IndexMetadata.INDEX_BLOCKS_SEARCH_ONLY_SETTING.getKey(), true).build() + ); + + throwable = assertThrows( + NoShardAvailableActionException.class, + () -> opRouting.searchShards(newState, indexNames, null, null, collector, outstandingRequests, null) + ); + + assertEquals( + String.format(Locale.ROOT, "The index %s is in the scale down state, and the number of search only shards is 0", "test0"), + throwable.getMessage() + ); + + IOUtils.close(clusterService); + terminate(threadPool); + } + // Regression test to ignore awareness attributes. This test creates shards in different zones and simulates stress // on nodes in one zone to test if Adapative Replica Selection smartly routes the request to a node in different zone // by ignoring the zone awareness attributes. @@ -1139,7 +1193,8 @@ public void testSearchReplicaDefaultRouting() throws Exception { indexNames, numShards, numReplicas, - numSearchReplicas + numSearchReplicas, + Settings.EMPTY ); IndexShardRoutingTable indexShardRoutingTable = state.getRoutingTable().index(indexName).getShards().get(0); ShardId shardId = indexShardRoutingTable.searchOnlyReplicas().get(0).shardId(); @@ -1216,7 +1271,8 @@ public void testSearchReplicaRoutingWhenSearchOnlyStrictSettingIsFalse() throws indexNames, numShards, numReplicas, - numSearchReplicas + numSearchReplicas, + Settings.EMPTY ); IndexShardRoutingTable indexShardRoutingTable = state.getRoutingTable().index(indexName).getShards().get(0); ShardId shardId = indexShardRoutingTable.searchOnlyReplicas().get(0).shardId(); diff --git a/test/framework/src/main/java/org/opensearch/action/support/replication/ClusterStateCreationUtils.java b/test/framework/src/main/java/org/opensearch/action/support/replication/ClusterStateCreationUtils.java index 0c4e871b1330c..bf24932c5f7b2 100644 --- a/test/framework/src/main/java/org/opensearch/action/support/replication/ClusterStateCreationUtils.java +++ b/test/framework/src/main/java/org/opensearch/action/support/replication/ClusterStateCreationUtils.java @@ -326,7 +326,7 @@ public static ClusterState stateWithAssignedPrimariesAndOneReplica(String index, * Creates cluster state with several indexes, shards and replicas and all shards STARTED. */ public static ClusterState stateWithAssignedPrimariesAndReplicas(String[] indices, int numberOfShards, int numberOfReplicas) { - return stateWithAssignedPrimariesAndReplicas(indices, numberOfShards, numberOfReplicas, 0); + return stateWithAssignedPrimariesAndReplicas(indices, numberOfShards, numberOfReplicas, 0, Settings.EMPTY); } /** @@ -336,7 +336,8 @@ public static ClusterState stateWithAssignedPrimariesAndReplicas( String[] indices, int numberOfShards, int numberOfReplicas, - int numberOfSearchReplicas + int numberOfSearchReplicas, + Settings indexSettings ) { int numberOfDataNodes = numberOfReplicas + 1; DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); @@ -361,6 +362,7 @@ public static ClusterState stateWithAssignedPrimariesAndReplicas( .put(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas) .put(SETTING_NUMBER_OF_SEARCH_REPLICAS, numberOfSearchReplicas) .put(SETTING_CREATION_DATE, System.currentTimeMillis()) + .put(indexSettings) ) .build(); metadataBuilder.put(indexMetadata, false).generateClusterUuidIfNeeded(); From 798e29e90ed1927b9ca8085e994f617b41ee4e41 Mon Sep 17 00:00:00 2001 From: guojialiang Date: Sat, 4 Apr 2026 11:58:46 +0800 Subject: [PATCH 2/2] update test. Signed-off-by: guojialiang --- .../indices/settings/SearchOnlyReplicaIT.java | 45 ------------------- 1 file changed, 45 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaIT.java index 58c4bee6ecf4f..db34bb113320d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaIT.java @@ -8,7 +8,6 @@ package org.opensearch.indices.settings; -import org.opensearch.action.NoShardAvailableActionException; import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.WriteRequest; @@ -26,7 +25,6 @@ import org.opensearch.test.OpenSearchIntegTestCase; import java.io.IOException; -import java.util.Locale; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; @@ -240,49 +238,6 @@ public void testSearchReplicaRoutingPreference() throws IOException { assertEquals(nodeId, indexShardRoutingTable.searchOnlyReplicas().get(0).currentNodeId()); } - public void testSearchReplicaRoutingPreferenceWithoutSearchReplicaShard() throws IOException { - int numSearchReplicas = 0; - int numWriterReplicas = 1; - internalCluster().startClusterManagerOnlyNode(); - internalCluster().startDataOnlyNode(); - createIndex( - TEST_INDEX, - Settings.builder() - .put(indexSettings()) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numWriterReplicas) - .put(IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS, numSearchReplicas) - .build() - ); - ensureYellow(TEST_INDEX); - client().prepareIndex(TEST_INDEX).setId("1").setSource("foo", "bar").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); - // add 2 nodes for the replicas - internalCluster().startDataOnlyNode(); - internalCluster().startSearchOnlyNode(); - - ensureGreen(TEST_INDEX); - - assertActiveShardCounts(numSearchReplicas, numWriterReplicas); - - assertHitCount(client().prepareSearch(TEST_INDEX).setPreference(null).setSize(0).get(), 1); - - Throwable throwable = assertThrows( - NoShardAvailableActionException.class, - () -> client().prepareSearch(TEST_INDEX) - .setPreference(Preference.SEARCH_REPLICA.type()) - .setQuery(QueryBuilders.matchAllQuery()) - .get() - ); - - assertEquals( - String.format( - Locale.ROOT, - "Strictly require querying search only shards, but the number of search only replicas for index %s is 0", - TEST_INDEX - ), - throwable.getMessage() - ); - } - public void testSearchReplicaRoutingPreferenceWhenSearchReplicaUnassigned() { internalCluster().startClusterManagerOnlyNode(); internalCluster().startDataOnlyNode();