[RW Separation] Provide clear information when a query exception occurs due to the number of search only shards being 0

server/src/internalClusterTest/java/org/opensearch/action/admin/indices/scale/searchonly/ScaleIndexIT.java

@@ -8,6 +8,7 @@
 
 package org.opensearch.action.admin.indices.scale.searchonly;
 
+import org.opensearch.action.NoShardAvailableActionException;
 import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.opensearch.action.index.IndexResponse;
 import org.opensearch.action.search.SearchResponse;
@@ -16,6 +17,7 @@
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.routing.IndexRoutingTable;
 import org.opensearch.cluster.routing.IndexShardRoutingTable;
+import org.opensearch.cluster.routing.Preference;
 import org.opensearch.cluster.routing.ShardRouting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.rest.RestStatus;
@@ -25,6 +27,7 @@
 import org.opensearch.test.OpenSearchIntegTestCase;
 
 import java.util.HashSet;
+import java.util.Locale;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
@@ -88,9 +91,17 @@ private void testFullLifecycle(int searchOnlyReplica) throws Exception {
                 assertSearchNodeDocCounts(10, TEST_INDEX);
             } else {
                 try {
-                    client().prepareSearch(TEST_INDEX).setSize(0).get();
-                } catch (Exception e) {
-                    assertTrue(e.getMessage().contains("all shards failed"));
+                    client().prepareSearch(TEST_INDEX).setPreference(Preference.SEARCH_REPLICA.type()).setSize(0).get();
+                    fail("search replica should have failed");
+                } catch (NoShardAvailableActionException e) {
+                    assertEquals(
+                        String.format(
+                            Locale.ROOT,
+                            "Strictly require querying search only shards, but the number of search only replicas for index %s is 0",
+                            TEST_INDEX
+                        ),
+                        e.getMessage()
+                    );
                 }
             }
         }, 30, TimeUnit.SECONDS);
@@ -124,6 +135,22 @@ private void testFullLifecycle(int searchOnlyReplica) throws Exception {
 
         ensureGreen(TEST_INDEX);
 
+        if (searchOnlyReplica == 0) {
+            try {
+                client().prepareSearch(TEST_INDEX).setSize(0).get();
+                fail("search replica should have failed");
+            } catch (NoShardAvailableActionException e) {
+                assertEquals(
+                    String.format(
+                        Locale.ROOT,
+                        "The index %s is in the scale down state, and the number of search only shards is 0",
+                        TEST_INDEX
+                    ),
+                    e.getMessage()
+                );
+            }
+        }
+
         assertAcked(
             client().admin()
                 .indices()

server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java

@@ -33,6 +33,7 @@
 package org.opensearch.cluster.routing;
 
 import org.apache.lucene.util.CollectionUtil;
+import org.opensearch.action.NoShardAvailableActionException;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.VirtualShardRoutingHelper;
@@ -59,6 +60,7 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -281,6 +283,29 @@ public GroupShardsIterator<ShardIterator> searchShards(
                 }
             }
 
+            if (indexMetadataForShard.getNumberOfSearchOnlyReplicas() == 0 && (Preference.SEARCH_REPLICA.type().equals(preference))) {
+                throw new NoShardAvailableActionException(
+                    null,
+                    String.format(
+                        Locale.ROOT,
+                        "Strictly require querying search only shards, but the number of search only replicas for index %s is 0",
+                        indexMetadataForShard.getIndex().getName()
+                    )
+                );
+            }
+
+            if (indexMetadataForShard.getNumberOfSearchOnlyReplicas() == 0
+                && indexMetadataForShard.getSettings().getAsBoolean(IndexMetadata.INDEX_BLOCKS_SEARCH_ONLY_SETTING.getKey(), false)) {
+                throw new NoShardAvailableActionException(
+                    null,
+                    String.format(
+                        Locale.ROOT,
+                        "The index %s is in the scale down state, and the number of search only shards is 0",
+                        indexMetadataForShard.getIndex().getName()
+                    )
+                );
+            }
+
             ShardIterator iterator = preferenceActiveShardIterator(
                 shard,
                 clusterState.nodes().getLocalNodeId(),

server/src/test/java/org/opensearch/cluster/routing/OperationRoutingTests.java

@@ -32,6 +32,7 @@
 package org.opensearch.cluster.routing;
 
 import org.opensearch.Version;
+import org.opensearch.action.NoShardAvailableActionException;
 import org.opensearch.action.support.replication.ClusterStateCreationUtils;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.metadata.IndexMetadata;
@@ -62,6 +63,7 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
@@ -672,6 +674,58 @@ public void testAdaptiveReplicaSelection() throws Exception {
         terminate(threadPool);
     }
 
+    public void testSearchExceptionWithoutSearchOnlyReplicas() throws Exception {
+        final int numIndices = 1;
+        final String[] indexNames = new String[numIndices];
+        for (int i = 0; i < numIndices; i++) {
+            indexNames[i] = "test" + i;
+        }
+        ClusterState state = ClusterStateCreationUtils.stateWithAssignedPrimariesAndReplicas(indexNames, 1, 1);
+        OperationRouting opRouting = new OperationRouting(
+            Settings.EMPTY,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
+        );
+        opRouting.setUseAdaptiveReplicaSelection(true);
+        TestThreadPool threadPool = new TestThreadPool("testThatOnlyNodesSupportNodeIds");
+        ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool);
+        ResponseCollectorService collector = new ResponseCollectorService(clusterService);
+        Map<String, Long> outstandingRequests = new HashMap<>();
+        Throwable throwable = assertThrows(
+            NoShardAvailableActionException.class,
+            () -> opRouting.searchShards(state, indexNames, null, Preference.SEARCH_REPLICA.type(), collector, outstandingRequests, null)
+        );
+
+        assertEquals(
+            String.format(
+                Locale.ROOT,
+                "Strictly require querying search only shards, but the number of search only replicas for index %s is 0",
+                "test0"
+            ),
+            throwable.getMessage()
+        );
+
+        ClusterState newState = ClusterStateCreationUtils.stateWithAssignedPrimariesAndReplicas(
+            indexNames,
+            1,
+            1,
+            0,
+            Settings.builder().put(IndexMetadata.INDEX_BLOCKS_SEARCH_ONLY_SETTING.getKey(), true).build()
+        );
+
+        throwable = assertThrows(
+            NoShardAvailableActionException.class,
+            () -> opRouting.searchShards(newState, indexNames, null, null, collector, outstandingRequests, null)
+        );
+
+        assertEquals(
+            String.format(Locale.ROOT, "The index %s is in the scale down state, and the number of search only shards is 0", "test0"),
+            throwable.getMessage()
+        );
+
+        IOUtils.close(clusterService);
+        terminate(threadPool);
+    }
+
     // Regression test to ignore awareness attributes. This test creates shards in different zones and simulates stress
     // on nodes in one zone to test if Adapative Replica Selection smartly routes the request to a node in different zone
     // by ignoring the zone awareness attributes.
@@ -1139,7 +1193,8 @@ public void testSearchReplicaDefaultRouting() throws Exception {
                 indexNames,
                 numShards,
                 numReplicas,
-                numSearchReplicas
+                numSearchReplicas,
+                Settings.EMPTY
             );
             IndexShardRoutingTable indexShardRoutingTable = state.getRoutingTable().index(indexName).getShards().get(0);
             ShardId shardId = indexShardRoutingTable.searchOnlyReplicas().get(0).shardId();
@@ -1216,7 +1271,8 @@ public void testSearchReplicaRoutingWhenSearchOnlyStrictSettingIsFalse() throws
                 indexNames,
                 numShards,
                 numReplicas,
-                numSearchReplicas
+                numSearchReplicas,
+                Settings.EMPTY
             );
             IndexShardRoutingTable indexShardRoutingTable = state.getRoutingTable().index(indexName).getShards().get(0);
             ShardId shardId = indexShardRoutingTable.searchOnlyReplicas().get(0).shardId();

test/framework/src/main/java/org/opensearch/action/support/replication/ClusterStateCreationUtils.java

@@ -326,7 +326,7 @@ public static ClusterState stateWithAssignedPrimariesAndOneReplica(String index,
      * Creates cluster state with several indexes, shards and replicas and all shards STARTED.
      */
     public static ClusterState stateWithAssignedPrimariesAndReplicas(String[] indices, int numberOfShards, int numberOfReplicas) {
-        return stateWithAssignedPrimariesAndReplicas(indices, numberOfShards, numberOfReplicas, 0);
+        return stateWithAssignedPrimariesAndReplicas(indices, numberOfShards, numberOfReplicas, 0, Settings.EMPTY);
     }
 
     /**
@@ -336,7 +336,8 @@ public static ClusterState stateWithAssignedPrimariesAndReplicas(
         String[] indices,
         int numberOfShards,
         int numberOfReplicas,
-        int numberOfSearchReplicas
+        int numberOfSearchReplicas,
+        Settings indexSettings
     ) {
         int numberOfDataNodes = numberOfReplicas + 1;
         DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder();
@@ -361,6 +362,7 @@ public static ClusterState stateWithAssignedPrimariesAndReplicas(
                         .put(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas)
                         .put(SETTING_NUMBER_OF_SEARCH_REPLICAS, numberOfSearchReplicas)
                         .put(SETTING_CREATION_DATE, System.currentTimeMillis())
+                        .put(indexSettings)
                 )
                 .build();
             metadataBuilder.put(indexMetadata, false).generateClusterUuidIfNeeded();