diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/scale/searchonly/ScaleIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/scale/searchonly/ScaleIndexIT.java index 97d0f8565394d..551cbfa077aa7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/scale/searchonly/ScaleIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/scale/searchonly/ScaleIndexIT.java @@ -8,6 +8,7 @@ package org.opensearch.action.admin.indices.scale.searchonly; +import org.opensearch.action.NoShardAvailableActionException; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.SearchResponse; @@ -16,6 +17,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; +import org.opensearch.cluster.routing.Preference; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.settings.Settings; import org.opensearch.core.rest.RestStatus; @@ -25,6 +27,7 @@ import org.opensearch.test.OpenSearchIntegTestCase; import java.util.HashSet; +import java.util.Locale; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -88,9 +91,17 @@ private void testFullLifecycle(int searchOnlyReplica) throws Exception { assertSearchNodeDocCounts(10, TEST_INDEX); } else { try { - client().prepareSearch(TEST_INDEX).setSize(0).get(); - } catch (Exception e) { - assertTrue(e.getMessage().contains("all shards failed")); + client().prepareSearch(TEST_INDEX).setPreference(Preference.SEARCH_REPLICA.type()).setSize(0).get(); + fail("search replica should have failed"); + } catch (NoShardAvailableActionException e) { + assertEquals( + String.format( + Locale.ROOT, + "Strictly require querying search only shards, but the number of search only replicas for index %s is 0", + TEST_INDEX + ), + e.getMessage() + ); } } }, 30, TimeUnit.SECONDS); @@ -124,6 +135,22 @@ private void testFullLifecycle(int searchOnlyReplica) throws Exception { ensureGreen(TEST_INDEX); + if (searchOnlyReplica == 0) { + try { + client().prepareSearch(TEST_INDEX).setSize(0).get(); + fail("search replica should have failed"); + } catch (NoShardAvailableActionException e) { + assertEquals( + String.format( + Locale.ROOT, + "The index %s is in the scale down state, and the number of search only shards is 0", + TEST_INDEX + ), + e.getMessage() + ); + } + } + assertAcked( client().admin() .indices() diff --git a/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java b/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java index 210e9828876df..4506bb0089d06 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java +++ b/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java @@ -33,6 +33,7 @@ package org.opensearch.cluster.routing; import org.apache.lucene.util.CollectionUtil; +import org.opensearch.action.NoShardAvailableActionException; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.VirtualShardRoutingHelper; @@ -59,6 +60,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -281,6 +283,29 @@ public GroupShardsIterator searchShards( } } + if (indexMetadataForShard.getNumberOfSearchOnlyReplicas() == 0 && (Preference.SEARCH_REPLICA.type().equals(preference))) { + throw new NoShardAvailableActionException( + null, + String.format( + Locale.ROOT, + "Strictly require querying search only shards, but the number of search only replicas for index %s is 0", + indexMetadataForShard.getIndex().getName() + ) + ); + } + + if (indexMetadataForShard.getNumberOfSearchOnlyReplicas() == 0 + && indexMetadataForShard.getSettings().getAsBoolean(IndexMetadata.INDEX_BLOCKS_SEARCH_ONLY_SETTING.getKey(), false)) { + throw new NoShardAvailableActionException( + null, + String.format( + Locale.ROOT, + "The index %s is in the scale down state, and the number of search only shards is 0", + indexMetadataForShard.getIndex().getName() + ) + ); + } + ShardIterator iterator = preferenceActiveShardIterator( shard, clusterState.nodes().getLocalNodeId(), diff --git a/server/src/test/java/org/opensearch/cluster/routing/OperationRoutingTests.java b/server/src/test/java/org/opensearch/cluster/routing/OperationRoutingTests.java index 1254c1a84e573..65dc494168e29 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/OperationRoutingTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/OperationRoutingTests.java @@ -32,6 +32,7 @@ package org.opensearch.cluster.routing; import org.opensearch.Version; +import org.opensearch.action.NoShardAvailableActionException; import org.opensearch.action.support.replication.ClusterStateCreationUtils; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; @@ -62,6 +63,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.TreeMap; @@ -672,6 +674,58 @@ public void testAdaptiveReplicaSelection() throws Exception { terminate(threadPool); } + public void testSearchExceptionWithoutSearchOnlyReplicas() throws Exception { + final int numIndices = 1; + final String[] indexNames = new String[numIndices]; + for (int i = 0; i < numIndices; i++) { + indexNames[i] = "test" + i; + } + ClusterState state = ClusterStateCreationUtils.stateWithAssignedPrimariesAndReplicas(indexNames, 1, 1); + OperationRouting opRouting = new OperationRouting( + Settings.EMPTY, + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); + opRouting.setUseAdaptiveReplicaSelection(true); + TestThreadPool threadPool = new TestThreadPool("testThatOnlyNodesSupportNodeIds"); + ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool); + ResponseCollectorService collector = new ResponseCollectorService(clusterService); + Map outstandingRequests = new HashMap<>(); + Throwable throwable = assertThrows( + NoShardAvailableActionException.class, + () -> opRouting.searchShards(state, indexNames, null, Preference.SEARCH_REPLICA.type(), collector, outstandingRequests, null) + ); + + assertEquals( + String.format( + Locale.ROOT, + "Strictly require querying search only shards, but the number of search only replicas for index %s is 0", + "test0" + ), + throwable.getMessage() + ); + + ClusterState newState = ClusterStateCreationUtils.stateWithAssignedPrimariesAndReplicas( + indexNames, + 1, + 1, + 0, + Settings.builder().put(IndexMetadata.INDEX_BLOCKS_SEARCH_ONLY_SETTING.getKey(), true).build() + ); + + throwable = assertThrows( + NoShardAvailableActionException.class, + () -> opRouting.searchShards(newState, indexNames, null, null, collector, outstandingRequests, null) + ); + + assertEquals( + String.format(Locale.ROOT, "The index %s is in the scale down state, and the number of search only shards is 0", "test0"), + throwable.getMessage() + ); + + IOUtils.close(clusterService); + terminate(threadPool); + } + // Regression test to ignore awareness attributes. This test creates shards in different zones and simulates stress // on nodes in one zone to test if Adapative Replica Selection smartly routes the request to a node in different zone // by ignoring the zone awareness attributes. @@ -1139,7 +1193,8 @@ public void testSearchReplicaDefaultRouting() throws Exception { indexNames, numShards, numReplicas, - numSearchReplicas + numSearchReplicas, + Settings.EMPTY ); IndexShardRoutingTable indexShardRoutingTable = state.getRoutingTable().index(indexName).getShards().get(0); ShardId shardId = indexShardRoutingTable.searchOnlyReplicas().get(0).shardId(); @@ -1216,7 +1271,8 @@ public void testSearchReplicaRoutingWhenSearchOnlyStrictSettingIsFalse() throws indexNames, numShards, numReplicas, - numSearchReplicas + numSearchReplicas, + Settings.EMPTY ); IndexShardRoutingTable indexShardRoutingTable = state.getRoutingTable().index(indexName).getShards().get(0); ShardId shardId = indexShardRoutingTable.searchOnlyReplicas().get(0).shardId(); diff --git a/test/framework/src/main/java/org/opensearch/action/support/replication/ClusterStateCreationUtils.java b/test/framework/src/main/java/org/opensearch/action/support/replication/ClusterStateCreationUtils.java index 0c4e871b1330c..bf24932c5f7b2 100644 --- a/test/framework/src/main/java/org/opensearch/action/support/replication/ClusterStateCreationUtils.java +++ b/test/framework/src/main/java/org/opensearch/action/support/replication/ClusterStateCreationUtils.java @@ -326,7 +326,7 @@ public static ClusterState stateWithAssignedPrimariesAndOneReplica(String index, * Creates cluster state with several indexes, shards and replicas and all shards STARTED. */ public static ClusterState stateWithAssignedPrimariesAndReplicas(String[] indices, int numberOfShards, int numberOfReplicas) { - return stateWithAssignedPrimariesAndReplicas(indices, numberOfShards, numberOfReplicas, 0); + return stateWithAssignedPrimariesAndReplicas(indices, numberOfShards, numberOfReplicas, 0, Settings.EMPTY); } /** @@ -336,7 +336,8 @@ public static ClusterState stateWithAssignedPrimariesAndReplicas( String[] indices, int numberOfShards, int numberOfReplicas, - int numberOfSearchReplicas + int numberOfSearchReplicas, + Settings indexSettings ) { int numberOfDataNodes = numberOfReplicas + 1; DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); @@ -361,6 +362,7 @@ public static ClusterState stateWithAssignedPrimariesAndReplicas( .put(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas) .put(SETTING_NUMBER_OF_SEARCH_REPLICAS, numberOfSearchReplicas) .put(SETTING_CREATION_DATE, System.currentTimeMillis()) + .put(indexSettings) ) .build(); metadataBuilder.put(indexMetadata, false).generateClusterUuidIfNeeded();