GET fallback for link checker

lib/link_checker.rb

@@ -0,0 +1,102 @@
+class LinkChecker
+  TIMEOUT = 5
+  attr_reader :log
+
+  def initialize(log: true)
+    @log = log
+    @cache = {}
+  end
+
+  def check(collection)
+    collection.find_each do |record|
+      check_record(record)
+    end
+  end
+
+  def check_record(record)
+    [record, *record.external_resources].each do |item|
+      next if item.url.blank?
+      code = @cache.fetch(item.url) { bad_response(item.url) } # Cache the result, there could be multiple resources linked to same URL
+      if code
+        puts "  #{code} - #{item.class.name} #{item.id}: #{item.url}" if log
+        if item.link_monitor
+          item.link_monitor.fail!(code)
+        else
+          item.create_link_monitor(url: item.url, code: code)
+        end
+      else
+        if item.link_monitor
+          item.link_monitor.success!
+        end
+      end
+    end
+  end
+
+  private
+
+  # The fake return codes on an exception are so the LinkMonitor object has something
+  # to store as "code" which might be tracked back to a particular problem.
+  def bad_response(url)
+    begin
+      host = URI.parse(url).host rescue nil
+      if @prev_host == host
+        n = rand(4) + 1 # Add some delay between requests to the same host to prevent flooding
+        sleep(n)
+      end
+      @prev_host = host
+      code = HTTParty.head(url, verify: false, open_timeout: TIMEOUT, read_timeout: TIMEOUT).code
+      code = get_code(url) if code == 400 || code == 405 # Try a GET if HEAD not allowed (or generic 400 error)
+      return nil if code >= 200 && code < 400 # Success or redirects are OK
+      return code
+    rescue EOFError => e
+      puts "  #{e.class.name}: #{e}" if log
+      return 490
+    rescue SocketError => e
+      puts "  #{e.class.name}: #{e}" if log
+      return 491
+    rescue Timeout::Error => e
+      puts "  #{e.class.name}: #{e}" if log
+      return 492
+    rescue StandardError => e
+      puts "  #{e.class.name}: #{e}" if log
+      return 493
+    end
+  end
+
+  # Gets a response code using a GET request, for the case where HEAD is not supported.
+  def get_code(url, redirect_limit: 5, open_timeout: TIMEOUT, read_timeout: TIMEOUT, use_range: true)
+    raise StandardError, 'too many redirects' if redirect_limit <= 0
+
+    uri = URI(url)
+
+    # Prepare the HTTP connection
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = (uri.scheme == 'https')
+    http.open_timeout = open_timeout
+    http.read_timeout = read_timeout
+    http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+
+    # Build GET request (not HEAD)
+    req = Net::HTTP::Get.new(uri)
+    req['Range'] = 'bytes=0-0' if use_range # fetch just the first byte if supported
+
+    http.start do |connection|
+      connection.request(req) do |res|
+        case res
+        when Net::HTTPRedirection
+          location = res['location']
+          new_uri = URI.join(uri, location).to_s
+          return get_code(new_uri,
+                          redirect_limit: redirect_limit - 1,
+                          open_timeout: open_timeout,
+                          read_timeout: read_timeout,
+                          use_range: use_range)
+        else
+          code = res.code.to_i
+          # Do not call res.read_body — we don't want to download the content
+          return code
+        end
+      end
+    end
+  end
+end

lib/tasks/check_urls.rake

@@ -7,97 +7,24 @@ namespace :tess do
 
   desc 'Check material URLs for dead links'
   task check_material_urls: :environment do
-    check_materials
+    puts 'Checking material URLs'
+    LinkChecker.new.check(Material)
   end
 
   desc 'Check event URLs for dead links'
   task check_event_urls: :environment do
-    check_events
+    puts 'Checking event URLs'
+    LinkChecker.new.check(Event)
   end
 
   desc 'Check event and material URLs for dead links'
   task check_resource_urls: :environment do
-    check_materials
-    check_events
-  end
-end
+    lc = LinkChecker.new
 
-def check_materials
-  puts 'Checking material URLs'
-  Material.find_each do |mat|
-    process_record(mat)
-  end
-end
+    puts 'Checking material URLs'
+    lc.check(Material)
 
-def check_events
-  puts 'Checking event URLs'
-  Event.find_each do |event|
-    process_record(event)
+    puts 'Checking event URLs'
+    lc.check(Event)
   end
 end
-
-def process_record(record)
-  if record.url
-    code = get_bad_response(record.url)
-    if code
-      puts "  #{code} - #{record.class.name} #{record.id}: #{record.url}"
-      if record.link_monitor
-        record.link_monitor.fail!(code)
-      else
-        record.create_link_monitor(url: record.url, code: code)
-      end
-    else
-      if record.link_monitor
-        record.link_monitor.success!
-      end
-    end
-  end
-
-  record.external_resources.each do |res|
-    next unless res.url
-
-    code = get_bad_response(res.url)
-
-    if code
-      puts "  #{code} - ExternalResource #{res.id}: #{res.url}"
-      if res.link_monitor
-        res.link_monitor.fail!(code)
-      else
-        res.create_link_monitor(url: res.url, code: code)
-      end
-    else
-      if res.link_monitor
-        res.link_monitor.success!
-      end
-    end
-  end
-end
-
-# The fake return codes on an exception are so the LinkMonitor object has something
-# to store as "code" which might be tracked back to a particular problem.
-def get_bad_response(url)
-  begin
-    host = URI.parse(url).host rescue nil
-    if @prev_host == host
-      n = rand(4) + 1
-      sleep(n)
-    end
-    @prev_host = host
-    response = HTTParty.head(url, verify: false)
-    return nil if response.code >= 200 && response.code < 400 # Success or redirects are OK
-    return response.code
-  rescue EOFError => e
-    puts "  #{e.class.name}: #{e}"
-    return 490
-  rescue SocketError => e
-    puts "  #{e.class.name}: #{e}"
-    return 491
-  rescue Net::ReadTimeout => e
-    puts "  #{e.class.name}: #{e}"
-    return 492
-  rescue StandardError => e
-    puts "  #{e.class.name}: #{e}"
-    return 493
-  end
-end
-

test/unit/link_checker_test.rb

@@ -0,0 +1,104 @@
+require 'test_helper'
+
+class LinkCheckerTest < ActiveSupport::TestCase
+
+  setup do
+    @link_checker = LinkChecker.new(log: false)
+  end
+
+  test 'check collection' do
+    event = events(:two)
+    WebMock.stub_request(:head, event.url).to_return(status: 200)
+    assert_nil event.link_monitor
+
+    assert_no_difference('LinkMonitor.count') do
+      @link_checker.check(Event.where(id: event.id))
+    end
+
+    assert_nil event.link_monitor
+  end
+
+  test 'check passing event url' do
+    event = events(:two)
+    WebMock.stub_request(:head, event.url).to_return(status: 200)
+    assert_nil event.link_monitor
+
+    assert_no_difference('LinkMonitor.count') do
+      @link_checker.check_record(event)
+    end
+
+    assert_nil event.link_monitor
+  end
+
+  test 'check failing event url' do
+    event = events(:two)
+    WebMock.stub_request(:head, event.url).to_return(status: 404)
+    assert_nil event.link_monitor
+
+    assert_difference('LinkMonitor.count', 1) do
+      @link_checker.check_record(event)
+    end
+
+    assert event.link_monitor
+    assert event.link_monitor.failed_at
+    assert_equal 404, event.link_monitor.code
+  end
+
+  test 'check failing event url that only responds to get' do
+    event = events(:two)
+    WebMock.stub_request(:head, event.url).to_return(status: 405)
+    WebMock.stub_request(:get, event.url).to_return(status: 200)
+    assert_nil event.link_monitor
+
+    assert_no_difference('LinkMonitor.count') do
+      @link_checker.check_record(event)
+    end
+
+    assert_nil event.link_monitor
+  end
+
+  test 'follows redirect' do
+    event = events(:two)
+    WebMock.stub_request(:head, event.url).to_return(status: 405)
+    WebMock.stub_request(:get, event.url).to_return(status: 302, headers: { location: 'http://website.com' })
+    WebMock.stub_request(:get, 'http://website.com').to_return(status: 404)
+    assert_nil event.link_monitor
+
+    assert_difference('LinkMonitor.count', 1) do
+      @link_checker.check_record(event)
+    end
+
+    assert event.link_monitor
+    assert event.link_monitor.failed_at
+    assert_equal 404, event.link_monitor.code
+  end
+
+  test 'avoids redirect loop' do
+    event = events(:two)
+    WebMock.stub_request(:head, event.url).to_return(status: 405)
+    WebMock.stub_request(:get, event.url).to_return(status: 302, headers: { location: 'http://website.com' })
+    WebMock.stub_request(:get, 'http://website.com').to_return(status: 302, headers: { location: event.url })
+    assert_nil event.link_monitor
+
+    assert_difference('LinkMonitor.count', 1) do
+      @link_checker.check_record(event)
+    end
+
+    assert event.link_monitor
+    assert event.link_monitor.failed_at
+    assert_equal 493, event.link_monitor.code
+  end
+
+  test 'check material with external resources' do
+    WebMock.stub_request(:head, 'http://myurl.com/123').to_return(status: 200)
+    WebMock.stub_request(:head, 'https://tess.elixir-uk.org/').to_return(status: 200)
+    WebMock.stub_request(:head, 'https://bio.tools/tool/SR-Tesseler').to_return(status: 404)
+    WebMock.stub_request(:head, 'https://fairsharing.org/bsg-p123456').to_return(status: 404)
+    material = materials(:material_with_external_resource)
+
+    assert_difference('LinkMonitor.count', 2) do
+      @link_checker.check_record(material)
+    end
+  end
+
+end