Merge pull request #964 from MITLibraries/dfs_query_then_fetch

Adds use_global_scoring feature

Author: JPrevost

app/graphql/types/query_type.rb

@@ -68,6 +68,9 @@ def record_id(id:, index:)
                                                    'of the words much match. Options include: "OR", "AND"'
       argument :query_mode, String, required: false, default_value: 'keyword',
                                     description: 'Search mode: "keyword" (lexical search), "semantic" (vector search), or "hybrid" (both)'
+      argument :use_global_scoring, Boolean, required: false, default_value: false,
+                                             description: 'Calculate relevance scores globally across all shards ' \
+                                                          'instead of per-shard. Defaults to false.'
 
       # applied filters
       argument :access_to_files_filter, [String],
@@ -105,12 +108,15 @@ def record_id(id:, index:)
     end
 
     def search(searchterm:, citation:, contributors:, funding_information:, geodistance:, geobox:, identifiers:,
-               locations:, subjects:, title:, index:, source:, from:, boolean_type:, fulltext:, per_page: 20, query_mode: 'keyword', **filters)
+               locations:, subjects:, title:, index:, source:, from:, boolean_type:, fulltext:, per_page: 20,
+               query_mode: 'keyword', use_global_scoring: false, **filters)
       query = construct_query(searchterm, citation, contributors, funding_information, geodistance, geobox, identifiers,
                               locations, subjects, title, source, boolean_type, filters, per_page, query_mode)
 
       results = Opensearch.new.search(from, query, Timdex::OSClient, highlight: highlight_requested?, index: index,
-                                                                     fulltext: fulltext, query_mode: query_mode, requested_aggregations: requested_aggregations)
+                                                                     fulltext: fulltext, query_mode: query_mode,
+                                                                     requested_aggregations: requested_aggregations,
+                                                                     use_global_scoring: use_global_scoring)
 
       response = {}
       response[:hits] = results['hits']['total']['value']

app/models/opensearch.rb

@@ -4,15 +4,16 @@ class Opensearch
   MAX_SIZE = 200
 
   def search(from, params, client, highlight: false, index: nil, fulltext: false, query_mode: 'keyword',
-             requested_aggregations: [])
+             requested_aggregations: [], use_global_scoring: false)
     @params = params
     @highlight = highlight
     @fulltext = fulltext?(fulltext)
     @query_mode = query_mode
     @requested_aggregations = requested_aggregations
     index = default_index unless index.present?
-    client.search(index:,
-                  body: build_query(from))
+    search_params = { index:, body: build_query(from) }
+    search_params[:search_type] = 'dfs_query_then_fetch' if use_global_scoring
+    client.search(**search_params)
   end
 
   # Only treat fulltext as true if it is boolean true or the string 'true' (case insensitive)

test/controllers/graphql_controller_test.rb

@@ -872,7 +872,7 @@ class GraphqlControllerTest < ActionDispatch::IntegrationTest
         initial_hits_count = json_dataset['data']['search']['hits']
         initial_still_images_count = json_dataset['data']['search']['aggregations']['contentType'].find do |x|
                                        x['key'] == 'still image'
-                                     end ['docCount']
+                                     end['docCount']
 
         post '/graphql', params: { query:
           '{
@@ -940,7 +940,7 @@ class GraphqlControllerTest < ActionDispatch::IntegrationTest
 
   test 'graphql search respects perPage argument' do
     VCR.use_cassette('opensearch_init') do
-      VCR.use_cassette('graphql_search_per_page_5', match_requests_on: [:method, :uri]) do
+      VCR.use_cassette('graphql_search_per_page_5', match_requests_on: %i[method uri]) do
         post '/graphql', params: { query: '{
                                     search(perPage:5) {
                                       hits
@@ -1140,4 +1140,88 @@ class GraphqlControllerTest < ActionDispatch::IntegrationTest
       end
     end
   end
+
+  test 'graphql search with useGlobalScoring true passes search_type to opensearch' do
+    mock_response = {
+      'hits' => {
+        'total' => { 'value' => 1 },
+        'hits' => [
+          {
+            '_source' => {
+              'title' => 'Data analytics and big data'
+            }
+          }
+        ]
+      }
+    }
+    # Verify that when useGlobalScoring is true, the search_type parameter is set
+    Opensearch.any_instance.expects(:search).with do |_from, _params, _client, **kwargs|
+      kwargs[:use_global_scoring] == true
+    end.returns(mock_response)
+
+    post '/graphql', params: { query: '{
+                                search(searchterm: "data analytics", useGlobalScoring: true) {
+                                  records {
+                                    title
+                                  }
+                                }
+                              }' }
+    assert_equal(200, response.status)
+  end
+
+  test 'graphql search with useGlobalScoring false passes use_global_scoring false to opensearch' do
+    mock_response = {
+      'hits' => {
+        'total' => { 'value' => 1 },
+        'hits' => [
+          {
+            '_source' => {
+              'title' => 'Data analytics and big data'
+            }
+          }
+        ]
+      }
+    }
+    # Verify that when useGlobalScoring is false (or omitted), use_global_scoring is false
+    Opensearch.any_instance.expects(:search).with do |_from, _params, _client, **kwargs|
+      kwargs[:use_global_scoring] == false
+    end.returns(mock_response)
+
+    post '/graphql', params: { query: '{
+                                search(searchterm: "data analytics", useGlobalScoring: false) {
+                                  records {
+                                    title
+                                  }
+                                }
+                              }' }
+    assert_equal(200, response.status)
+  end
+
+  test 'graphql search useGlobalScoring defaults to false' do
+    mock_response = {
+      'hits' => {
+        'total' => { 'value' => 1 },
+        'hits' => [
+          {
+            '_source' => {
+              'title' => 'Data analytics and big data'
+            }
+          }
+        ]
+      }
+    }
+    # Verify that when useGlobalScoring is not specified, use_global_scoring defaults to false
+    Opensearch.any_instance.expects(:search).with do |_from, _params, _client, **kwargs|
+      kwargs[:use_global_scoring] == false
+    end.returns(mock_response)
+
+    post '/graphql', params: { query: '{
+                                search(searchterm: "data analytics") {
+                                  records {
+                                    title
+                                  }
+                                }
+                              }' }
+    assert_equal(200, response.status)
+  end
 end

test/models/opensearch_test.rb

@@ -202,4 +202,34 @@ class OpensearchTest < ActiveSupport::TestCase
     assert json['aggregations'].key?('source')
     assert_not json['aggregations'].key?('invalid_agg')
   end
+
+  test 'search with use_global_scoring false does not set search_type' do
+    mock_client = mock
+    mock_client.expects(:search).with do |params|
+      !params.key?(:search_type)
+    end.returns({ 'hits' => { 'hits' => [], 'total' => { 'value' => 0 } } })
+
+    os = Opensearch.new
+    os.search(0, {}, mock_client, use_global_scoring: false)
+  end
+
+  test 'search with use_global_scoring true sets search_type to dfs_query_then_fetch' do
+    mock_client = mock
+    mock_client.expects(:search).with do |params|
+      params[:search_type] == 'dfs_query_then_fetch'
+    end.returns({ 'hits' => { 'hits' => [], 'total' => { 'value' => 0 } } })
+
+    os = Opensearch.new
+    os.search(0, {}, mock_client, use_global_scoring: true)
+  end
+
+  test 'search defaults to use_global_scoring false' do
+    mock_client = mock
+    mock_client.expects(:search).with do |params|
+      !params.key?(:search_type)
+    end.returns({ 'hits' => { 'hits' => [], 'total' => { 'value' => 0 } } })
+
+    os = Opensearch.new
+    os.search(0, {}, mock_client)
+  end
 end