Merge branch 'main' into dave-testing-matomo-code

Author: djanelle-mit

.env.test

@@ -1,4 +1,6 @@
 ALMA_OPENURL=https://na06.alma.exlibrisgroup.com/view/uresolver/01MIT_INST/openurl?
+TURNSTILE_SITEKEY=test-sitekey
+TURNSTILE_SECRET=test-secret
 FEATURE_TIMDEX_FULLTEXT=true
 FEATURE_GEODATA=false
 MIT_PRIMO_URL=https://mit.primo.exlibrisgroup.com

Gemfile

@@ -4,6 +4,7 @@ git_source(:github) { |repo| "https://github.com/#{repo}.git" }
 ruby '3.4.8'
 
 gem 'bootsnap', require: false
+gem 'crawler_detect'
 gem 'graphql'
 gem 'graphql-client'
 gem 'http'
@@ -14,6 +15,7 @@ gem 'openssl'
 gem 'puma'
 gem 'rack-attack'
 gem 'rack-timeout'
+gem 'rails_cloudflare_turnstile'
 gem 'rails', '~> 7.2.0'
 gem 'redis'
 gem 'scout_apm'

Gemfile.lock

@@ -120,6 +120,8 @@ GEM
       bigdecimal
       rexml
     crass (1.0.6)
+    crawler_detect (1.2.9)
+      qonfig (>= 0.24)
     date (3.5.1)
     debug (1.11.1)
       irb (~> 1.10)
@@ -134,6 +136,12 @@ GEM
     drb (2.2.3)
     erb (5.1.3)
     erubi (1.13.1)
+    faraday (2.14.1)
+      faraday-net_http (>= 2.0, < 3.5)
+      json
+      logger
+    faraday-net_http (3.4.2)
+      net-http (~> 0.5)
     ffi (1.17.2-aarch64-linux-gnu)
     ffi (1.17.2-arm64-darwin)
     ffi (1.17.2-x86_64-darwin)
@@ -206,6 +214,8 @@ GEM
     mocha (2.8.2)
       ruby2_keywords (>= 0.0.5)
     msgpack (1.8.0)
+    net-http (0.9.1)
+      uri (>= 0.11.1)
     net-imap (0.5.13)
       date
       net-protocol
@@ -243,6 +253,8 @@ GEM
     public_suffix (6.0.2)
     puma (7.2.0)
       nio4r (~> 2.0)
+    qonfig (0.30.0)
+      base64 (>= 0.2)
     racc (1.8.1)
     rack (3.1.20)
     rack-attack (6.8.0)
@@ -276,6 +288,9 @@ GEM
     rails-html-sanitizer (1.7.0)
       loofah (~> 2.25)
       nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0)
+    rails_cloudflare_turnstile (0.5.0)
+      faraday (>= 1.0, < 3.0)
+      rails (>= 6.0, < 8.2)
     railties (7.2.3)
       actionpack (= 7.2.3)
       activesupport (= 7.2.3)
@@ -381,6 +396,7 @@ GEM
     unicode-display_width (3.2.0)
       unicode-emoji (~> 4.1)
     unicode-emoji (4.2.0)
+    uri (1.1.1)
     useragent (0.16.11)
     vcr (6.4.0)
     web-console (4.2.1)
@@ -421,6 +437,7 @@ DEPENDENCIES
   bootsnap
   capybara
   climate_control
+  crawler_detect
   debug
   dotenv-rails
   graphql
@@ -437,6 +454,7 @@ DEPENDENCIES
   rack-attack
   rack-timeout
   rails (~> 7.2.0)
+  rails_cloudflare_turnstile
   redis
   rubocop
   rubocop-rails

README.md

@@ -95,6 +95,7 @@ See `Optional Environment Variables` for more information.
 - `BOOLEAN_OPTIONS`: comma separated list of values to present to testers on instances where `BOOLEAN_PICKER` feature is enabled.
 - `FEATURE_BOOLEAN_PICKER`: feature to allow users to select their preferred boolean type. If set to `true`, feature is enabled. This feature is only intended for internal team
   testing and should never be enabled in production (mostly because the UI is a mess more than it would cause harm).
+- `FEATURE_BOT_DETECTION`: When set to `true`, enables bot detection using crawler_detect and Cloudflare Turnstile challenges for suspected bots on search result pages. Requires `TURNSTILE_SITEKEY` and `TURNSTILE_SECRET` to be set. If disabled, bots may crawl search results freely.
 - `FEATURE_GEODATA`: Enables features related to geospatial data discovery. Setting this variable to `true` will trigger geodata
 mode. Note that this is currently intended _only_ for the geodata app and
 may have unexpected consequences if applied to other TIMDEX UI apps.
@@ -146,6 +147,8 @@ instance is sending what search traffic. Defaults to "unset" if not defined.
 - `TIMDEX_INDEX`: Name of the index, or alias, to provide to the GraphQL endpoint. Defaults to `nil` which will let TIMDEX determine the best index to use. Wildcard values can be set, for example `rdi*` would search any indexes that begin with `rdi` in the underlying OpenSearch instance behind TIMDEX.
 - `TIMDEX_SOURCES`: Comma-separated list of sources to display in the advanced-search source selection element. This
   overrides the default which is set in ApplicationHelper.
+- `TURNSTILE_SECRET`: The Cloudflare Turnstile secret key used to verify challenge responses. If not set, bot challenge protection is disabled.
+- `TURNSTILE_SITEKEY`: The Cloudflare Turnstile site key used to render the challenge widget. If not set, bot challenge protection is disabled.
 
 #### Test Environment-only Variables
 

app/assets/stylesheets/partials/_pagination.scss

@@ -6,6 +6,7 @@
   padding-top: 1em;
   border-top: 1px solid $color-border-default;
   gap: 24px;
+  margin-bottom: 48px;
 
   .center-elements {
     display: flex;

app/assets/stylesheets/partials/_results.scss

@@ -397,7 +397,7 @@
 }
 
 // -----------------------------
-// Result list and Sidebar - USE
+// Result list - USE
 // -----------------------------
 
 #content-wrapper {
@@ -428,77 +428,18 @@
       flex-grow: 1;
     }
 
-    #results-sidebar {
-      flex: 0 0 360px;
-
-      display: flex;
-      flex-direction: column;
-      gap: 32px;
-
-      > div {
-        display: flex;
-        gap: 12px;
-
-        i {
-          font-size: 2rem;
-          margin-top: 1px;
-        }
-      }
-
-      @media (max-width: 1140px) {
-        flex-basis: 280px;
-      }
-
-      p {
-        line-height: 1.4;
-      }
-
-      a {
-        @include underlinedLinks;
-        font-weight: $fw-medium;   
-      }
-    }
-
     // Put the sidebar below the results when the screen is narrower than 1024px
     @media (max-width: $bp-screen-lg) {
         flex-direction: column;
-        gap: 32px;
-
-        // Put the sidebar items side by side when space is available
-        #results-sidebar {
-          flex-direction: row;
-          padding-bottom: 40px;
-          gap: 12px;
-          flex-basis: auto;
-
-          > div {
-            padding-right: 24px;
-          }
-        }
-
-        #callout-wrapper {
-          margin-bottom: 0;
-        }
-    }
-
-    // Put sidebar items in a column below 768px
-    @media (max-width: $bp-screen-md) {
-    
-      #results-sidebar {
-        flex-direction: column;
-        gap: 32px;
-      }
-      
+        gap: 0px;
     }
-  }
-
 }
 
 // ----------------------------
 // Result callout boxes
 // ----------------------------
 
-#callout-wrapper {
+.callout-wrapper {
   display: flex;
   gap: 12px;
   margin-bottom: 32px;
@@ -514,6 +455,7 @@
     i {
       font-size: 2rem;
       margin-top: 1px;
+      flex: 0 0 20px;
     }
 
     p {
@@ -529,3 +471,99 @@
     flex-direction: column;
   }
 }
+
+// Callout box responsive behavior
+// SIDEBAR COLLAPSES BELOW RESULTS
+@media (max-width: $bp-screen-lg) {
+  // Hide the bordered callouts
+  #results > .callout-wrapper {
+    display: none;
+  }
+}
+
+
+// -----------------------------
+// Result sidebar
+// -----------------------------
+
+ #results-layout-wrapper #results-sidebar {
+    flex: 0 0 360px;
+
+    display: flex;
+    flex-direction: column;
+    gap: 32px;
+
+    .core-sidebar-items {
+      display: flex;
+      flex-direction: column;
+      gap: 32px;
+
+      > div {
+        display: flex;
+        gap: 12px;
+
+        i {
+          font-size: 2rem;
+          margin-top: 1px;
+          flex: 0 0 20px;
+        }
+      }
+    }
+
+    @media (max-width: 1140px) {
+      flex-basis: 280px;
+    }
+
+    p {
+      line-height: 1.4;
+    }
+
+    a {
+      @include underlinedLinks;
+      font-weight: $fw-medium;   
+    }
+
+    // Override some of the styling of the callout boxes when they're in the sidebar.
+    .callout-wrapper {
+      aside {
+        border: none;
+        padding: 0;
+
+        a {
+          text-decoration: none;
+        }
+      }
+
+      flex-direction: column;
+      gap: 32px;
+      border-bottom: 1px solid $color-border-default;
+      padding-bottom: 32px;
+      margin-bottom: 0;
+    }  
+
+    // Re-layout the sidebar contents when below results on smaller screens
+    @media (max-width: $bp-screen-lg) {
+      
+      flex-direction: column;
+
+      .callout-wrapper {
+        flex-direction: row;
+      }
+
+      .core-sidebar-items {
+        flex-direction: row;
+      }
+
+    }
+
+    @media (max-width: $bp-screen-sm2) {
+      .callout-wrapper {
+        flex-direction: column;
+      }
+
+      .core-sidebar-items {
+        flex-direction: column;
+      }      
+    }
+  }
+}

app/assets/stylesheets/partials/_variables.scss

@@ -116,6 +116,7 @@ $container-max-width: 1280px;
 
 // Additional breakpoints over mitlib-style
 $bp-screen-xs: 410px;
+$bp-screen-sm2: 640px;
 
 // Left/Right Edge Padding
 $edge-padding-desktop: 32px;

app/controllers/search_controller.rb

@@ -2,6 +2,7 @@ class SearchController < ApplicationController
   before_action :validate_q!, only: %i[results]
   before_action :validate_format_token, only: %i[results]
   before_action :set_active_tab, only: %i[results]
+  before_action :challenge_bots!, only: %i[results]
   around_action :sleep_if_too_fast, only: %i[results]
 
   before_action :validate_geobox_presence!, only: %i[results]
@@ -271,6 +272,15 @@ def validate_q!
     redirect_to root_url
   end
 
+  # Redirect suspected crawlers to Turnstile when the bot_detection feature is enabled.
+  def challenge_bots!
+    return unless Feature.enabled?(:bot_detection)
+    return if session[:passed_turnstile]
+    return unless BotDetector.should_challenge?(request)
+
+    redirect_to turnstile_path(return_to: request.fullpath)
+  end
+
   def validate_geodistance_presence!
     return unless Feature.enabled?(:geodata)
 

app/controllers/turnstile_controller.rb

@@ -0,0 +1,33 @@
+class TurnstileController < ApplicationController
+  before_action :validate_cloudflare_turnstile, only: :verify
+
+  rescue_from RailsCloudflareTurnstile::Forbidden, with: :handle_forbidden
+
+  def show
+    @return_to = params[:return_to].presence || root_path
+  end
+
+  def verify
+    session[:passed_turnstile] = true
+    redirect_to safe_return_path
+  end
+
+  private
+
+  # Handles Turnstile rejecting token submission due to invalid token, network issue, etc.
+  def handle_forbidden
+    flash.now[:error] = "We couldn't complete the verification. Please try again."
+    render :show, status: :unprocessable_entity
+  end
+
+  # Returns a safe path to redirect to after Turnstile verification. Valid paths should begin with
+  # a single slash. Falls back to root_path if the provided path is invalid.
+  def safe_return_path
+    return_to = params[:return_to].to_s
+    return root_path if return_to.blank?
+    return root_path if return_to.start_with?('//')
+    return return_to if return_to.start_with?('/')
+
+    root_path
+  end
+end

app/javascript/filters.js

@@ -3,6 +3,12 @@ function initFilterToggle() {
   var filter_toggle = document.getElementById('filter-toggle');
   var filter_panel = document.getElementById('filter-container');
   var filter_categories = document.getElementsByClassName('filter-category');
+
+  // No need for event listeners if filters aren't present.
+  if (!filter_toggle || !filter_panel) {
+    return;
+  }
+
   filter_toggle.addEventListener('click', event => {
     filter_panel.classList.toggle('hidden-md');
     filter_toggle.classList.toggle('expanded');