Add feature to enable semantic search

jazairi · jazairi · commit 58f7bd5813ce · 2026-04-17T08:59:14.000-07:00
Why these changes are being introduced: We need a means to toggle the new semantic search query mode in the UI. Relevant ticket(s): - [USE-493](https://mitlibraries.atlassian.net/browse/USE-493) How this addresses that need: This adds a feature to toggle semantic search on and off. Lexical search remains the default. Side effects of this change: - This feature is explicitly disabled for geospatial queries. If we want semantic search for GeoData in the future, we will need to revisit the code. - There is no query param exposing this feature, so it is not currently possible to toggle on a per-query basis.
diff --git a/README.md b/README.md
@@ -106,6 +106,7 @@ may have unexpected consequences if applied to other TIMDEX UI apps.
 - `FEATURE_TAB_TIMDEX_ALL`: Display a tab for displaying the combined TIMDEX data. `TIMDEX_INDEX` affects which data appears in this tab.
 - `FEATURE_TAB_TIMDEX_ALMA`: Display a tab for displaying Alma data from TIMDEX. `TIMDEX_INDEX` must include `Alma` data or no results will return.
 - `FEATURE_TIMDEX_FULLTEXT`: Activate fulltext searching for sources in TIMDEX that support it
+- `FEATURE_TIMDEX_SEMANTIC_SEARCH`: Enables semantic query mode (`queryMode: semantic`) for TIMDEX searches. When disabled, TIMDEX defaults to lexical search behavior.
 - `FEATURE_PRIMO_NDE_LINKS`: Enables all Primo UI links to target the NDE version of Primo. When enabled, links will use `/nde/search` and `/nde/fulldisplay` paths along with the NDE view ID from `PRIMO_NDE_VID`.
 - `FILTER_ACCESS_TO_FILES`: The name to use instead of "Access to files" for that filter / aggregation.
 - `FILTER_CONTENT_TYPE`: The name to use instead of "Content type" for that filter / aggregation.
diff --git a/app/controllers/search_controller.rb b/app/controllers/search_controller.rb
@@ -193,7 +193,7 @@ def query_timdex(query)
       raw = if Feature.enabled?(:geodata)
               execute_geospatial_query(query)
             else
-              TimdexBase::Client.query(TimdexSearch::BaseQuery, variables: query)
+              TimdexBase::Client.query(base_query_for_mode, variables: query)
             end
 
       # The response type is a GraphQL::Client::Response, which is not directly serializable, so we
@@ -217,17 +217,25 @@ def query_primo(per_page, offset)
   end
 
   def execute_geospatial_query(query)
+    query = query.except('queryMode')
+
     if query['geobox'] == 'true' && query[:geodistance] == 'true'
       TimdexBase::Client.query(TimdexSearch::AllQuery, variables: query)
     elsif query['geobox'] == 'true'
       TimdexBase::Client.query(TimdexSearch::GeoboxQuery, variables: query)
     elsif query['geodistance'] == 'true'
       TimdexBase::Client.query(TimdexSearch::GeodistanceQuery, variables: query)
     else
-      TimdexBase::Client.query(TimdexSearch::BaseQuery, variables: query)
+      TimdexBase::Client.query(base_query_for_mode, variables: query)
     end
   end
 
+  def base_query_for_mode
+    return TimdexSearch::BaseQuery unless Feature.enabled?(:timdex_semantic_search)
+
+    TimdexSearch::SemanticBaseQuery
+  end
+
   def extract_errors(response)
     response[:errors]['data'] if response.is_a?(Hash) && response.key?(:errors) && response[:errors].key?('data')
   end
diff --git a/app/models/feature.rb b/app/models/feature.rb
@@ -34,7 +34,7 @@
 class Feature
   # List of all valid features in the application
   VALID_FEATURES = %i[bot_detection geodata boolean_picker oa_always primo_nde_links simulate_search_latency tab_primo_all tab_timdex_all
-                      tab_timdex_alma record_link timdex_fulltext].freeze
+                      tab_timdex_alma record_link timdex_fulltext timdex_semantic_search].freeze
 
   # Check if a feature is enabled by name
   #
diff --git a/app/models/query_builder.rb b/app/models/query_builder.rb
@@ -20,6 +20,7 @@ def initialize(enhanced_query)
     extract_query(enhanced_query)
     extract_geosearch(enhanced_query)
     extract_filters(enhanced_query)
+    @query['queryMode'] = 'semantic' if Feature.enabled?(:timdex_semantic_search)
     @query['index'] = ENV.fetch('TIMDEX_INDEX', nil)
     @query['booleanType'] = enhanced_query[:booleanType]
     @query.compact!
diff --git a/app/models/timdex_search.rb b/app/models/timdex_search.rb
@@ -144,6 +144,150 @@ class TimdexSearch < TimdexBase
     }
   GRAPHQL
 
+  SemanticBaseQuery = TimdexBase::Client.parse <<-GRAPHQL
+    query(
+      $q: String
+      $citation: String
+      $contributors: String
+      $fundingInformation: String
+      $identifiers: String
+      $locations: String
+      $subjects: String
+      $title: String
+      $index: String
+      $from: String
+      $booleanType: String
+      $queryMode: String
+      $fulltext: Boolean
+      $perPage: Int
+      $accessToFilesFilter: [String!]
+      $contentTypeFilter: [String!]
+      $contributorsFilter: [String!]
+      $formatFilter: [String!]
+      $languagesFilter: [String!]
+      $literaryFormFilter: String
+      $placesFilter: [String!]
+      $sourceFilter: [String!]
+      $subjectsFilter: [String!]
+    ) {
+      search(
+        searchterm: $q
+        citation: $citation
+        contributors: $contributors
+        fundingInformation: $fundingInformation
+        identifiers: $identifiers
+        locations: $locations
+        subjects: $subjects
+        title: $title
+        index: $index
+        from: $from
+        booleanType: $booleanType
+        queryMode: $queryMode
+        fulltext: $fulltext
+        perPage: $perPage
+        accessToFilesFilter: $accessToFilesFilter
+        contentTypeFilter: $contentTypeFilter
+        contributorsFilter: $contributorsFilter
+        formatFilter: $formatFilter
+        languagesFilter: $languagesFilter
+        literaryFormFilter: $literaryFormFilter
+        placesFilter: $placesFilter
+        sourceFilter: $sourceFilter
+        subjectsFilter: $subjectsFilter
+      ) {
+        hits
+        records {
+          timdexRecordId
+          identifiers {
+            kind
+            value
+          }
+          title
+          source
+          contentType
+          contributors {
+            kind
+            value
+          }
+          publicationInformation
+          dates {
+            kind
+            value
+            range {
+              gte
+              lte
+            }
+          }
+          links {
+            kind
+            restrictions
+            text
+            url
+          }
+          notes {
+            kind
+            value
+          }
+          highlight {
+            matchedField
+            matchedPhrases
+          }
+          provider
+          rights {
+            kind
+            description
+            uri
+          }
+          sourceLink
+          summary
+          subjects {
+            kind
+            value
+          }
+          citation
+        }
+        aggregations {
+          accessToFiles {
+            key
+            docCount
+          }
+          contentType {
+            key
+            docCount
+          }
+          contributors {
+            key
+            docCount
+          }
+          format {
+            key
+            docCount
+          }
+          languages {
+            key
+            docCount
+          }
+          literaryForm {
+            key
+            docCount
+          }
+          places {
+            key
+            docCount
+          }
+          source {
+            key
+            docCount
+          }
+          subjects {
+            key
+            docCount
+          }
+        }
+      }
+    }
+  GRAPHQL
+
   GeoboxQuery = TimdexBase::Client.parse <<-GRAPHQL
     query(
       $q: String
diff --git a/config/schema/schema.json b/config/schema/schema.json
@@ -1360,6 +1360,16 @@
                   },
                   "defaultValue": "\"OR\""
                 },
+                {
+                  "name": "queryMode",
+                  "description": "Search mode to use. Defaults to \"lexical\". Options include: \"lexical\", \"semantic\"",
+                  "type": {
+                    "kind": "SCALAR",
+                    "name": "String",
+                    "ofType": null
+                  },
+                  "defaultValue": "\"lexical\""
+                },
                 {
                   "name": "accessToFilesFilter",
                   "description": "Filter results by access type. Use the `AccessToFiles` aggregation for a list of possible values. Multiple values are ORed.",
diff --git a/test/controllers/search_controller_test.rb b/test/controllers/search_controller_test.rb
@@ -1220,4 +1220,26 @@ def source_filter_count(controller)
     # Should not be redirected to Turnstile (doesn't hit SearchController)
     assert_response :success
   end
+
+  test 'uses BaseQuery when semantic search feature is disabled' do
+    # When the feature flag is not enabled, base_query_for_mode returns BaseQuery (default tab is 'all')
+    mock_primo_search_all_tab
+    mock_timdex_search_all_tab
+
+    get '/results?q=test'
+
+    assert_response :success
+  end
+
+  test 'uses SemanticBaseQuery when semantic search feature is enabled' do
+    # When the feature flag is enabled, base_query_for_mode returns SemanticBaseQuery (default tab is 'all')
+    ClimateControl.modify FEATURE_TIMDEX_SEMANTIC_SEARCH: 'true' do
+      mock_primo_search_all_tab
+      mock_timdex_search_all_tab
+
+      get '/results?q=test'
+
+      assert_response :success
+    end
+  end
 end
diff --git a/test/models/query_builder_test.rb b/test/models/query_builder_test.rb
@@ -120,4 +120,16 @@ class QueryBuilderTest < ActiveSupport::TestCase
     }
     assert_equal expected, QueryBuilder.new(search).query
   end
+
+  test 'query builder defaults to lexical mode by omitting queryMode' do
+    search = { q: 'blah' }
+    refute_includes(QueryBuilder.new(search).query.keys, 'queryMode')
+  end
+
+  test 'query builder adds semantic queryMode when feature flag is enabled' do
+    ClimateControl.modify FEATURE_TIMDEX_SEMANTIC_SEARCH: 'true' do
+      search = { q: 'blah' }
+      assert_equal('semantic', QueryBuilder.new(search).query['queryMode'])
+    end
+  end
 end
diff --git a/test_output.log b/test_output.log
@@ -0,0 +1,15 @@
+Running 582 tests in a single process (parallelization threshold is 999)
+Run options: --seed 1257
+
+# Running:
+
+............................................................................................................................................................................................................................................................S........................................................................................................................................................................................................................S.S..S....SS.....S.S.S......SS.............S.S..........S..SS..S.................................................
+
+Finished in 2.615215s, 222.5438 runs/s, 584.6556 assertions/s.
+
+582 runs, 1529 assertions, 0 failures, 0 errors, 17 skips
+
+You have skipped tests. Run with --verbose for details.
+Coverage report generated for Minitest, Unit Tests to /Users/jazairi/workspace/timdex-ui/coverage.
+Line Coverage: 96.67% (1366 / 1413)
+Lcov style coverage report generated for Minitest, Unit Tests to /Users/jazairi/workspace/timdex-ui/coverage/lcov/coverage.lcov

Original file line number	Diff line number	Diff line change
`@@ -34,7 +34,7 @@`
`34`	`34`	`class Feature`
`35`	`35`	`# List of all valid features in the application`
`36`	`36`	`VALID_FEATURES = %i[bot_detection geodata boolean_picker oa_always primo_nde_links simulate_search_latency tab_primo_all tab_timdex_all`
`37`		`- tab_timdex_alma record_link timdex_fulltext].freeze`
	`37`	`+ tab_timdex_alma record_link timdex_fulltext timdex_semantic_search].freeze`
`38`	`38`
`39`	`39`	`# Check if a feature is enabled by name`
`40`	`40`	`#`