Skip to content

Commit 8465f11

Browse files
fbacallfbacall
committed
Fallback to a GET request if HEAD fails when checking links
1 parent 5dc63a2 commit 8465f11

3 files changed

Lines changed: 176 additions & 68 deletions

File tree

‎lib/link_checker.rb‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
class LinkChecker
2+
def self.check(record, log: true)
3+
[record, *record.external_resources].each do |item|
4+
next if item.url.blank?
5+
code = bad_response(item.url, log: log)
6+
if code
7+
puts " #{code} - #{item.class.name} #{item.id}: #{item.url}" if log
8+
if item.link_monitor
9+
item.link_monitor.fail!(code)
10+
else
11+
item.create_link_monitor(url: item.url, code: code)
12+
end
13+
else
14+
if item.link_monitor
15+
item.link_monitor.success!
16+
end
17+
end
18+
end
19+
end
20+
21+
# The fake return codes on an exception are so the LinkMonitor object has something
22+
# to store as "code" which might be tracked back to a particular problem.
23+
def self.bad_response(url, log: true)
24+
begin
25+
host = URI.parse(url).host rescue nil
26+
if @prev_host == host
27+
n = rand(4) + 1 # Add some delay between requests to the same host to prevent flooding
28+
sleep(n)
29+
end
30+
@prev_host = host
31+
code = HTTParty.head(url, verify: false).code
32+
code = get_code(url) if code == 400 || code == 405 # Try a GET if HEAD not allowed (or generic 400 error)
33+
return nil if code >= 200 && code < 400 # Success or redirects are OK
34+
return code
35+
rescue EOFError => e
36+
puts " #{e.class.name}: #{e}" if log
37+
return 490
38+
rescue SocketError => e
39+
puts " #{e.class.name}: #{e}" if log
40+
return 491
41+
rescue Timeout::Error => e
42+
puts " #{e.class.name}: #{e}" if log
43+
return 492
44+
rescue StandardError => e
45+
puts " #{e.class.name}: #{e}" if log
46+
return 493
47+
end
48+
end
49+
50+
# Gets a response code using a GET request, for the case where HEAD is not supported.
51+
def self.get_code(url, redirect_limit: 5, open_timeout: 5, read_timeout: 5, use_range: true)
52+
raise StandardError, 'too many redirects' if redirect_limit <= 0
53+
54+
uri = URI(url)
55+
56+
# Prepare the HTTP connection
57+
http = Net::HTTP.new(uri.host, uri.port)
58+
http.use_ssl = (uri.scheme == 'https')
59+
http.open_timeout = open_timeout
60+
http.read_timeout = read_timeout
61+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
62+
63+
# Build GET request (not HEAD)
64+
req = Net::HTTP::Get.new(uri)
65+
req['Range'] = 'bytes=0-0' if use_range # fetch just the first byte if supported
66+
67+
http.start do |connection|
68+
connection.request(req) do |res|
69+
case res
70+
when Net::HTTPRedirection
71+
location = res['location']
72+
new_uri = URI.join(uri, location).to_s
73+
return get_code(new_uri,
74+
redirect_limit: redirect_limit - 1,
75+
open_timeout: open_timeout,
76+
read_timeout: read_timeout,
77+
use_range: use_range)
78+
else
79+
code = res.code.to_i
80+
# Do not call res.read_body — we don't want to download the content
81+
return code
82+
end
83+
end
84+
end
85+
end
86+
end

‎lib/tasks/check_urls.rake‎

Lines changed: 2 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -25,79 +25,13 @@ end
2525
def check_materials
2626
puts 'Checking material URLs'
2727
Material.find_each do |mat|
28-
process_record(mat)
28+
LinkChecker.check(mat)
2929
end
3030
end
3131

3232
def check_events
3333
puts 'Checking event URLs'
3434
Event.find_each do |event|
35-
process_record(event)
35+
LinkChecker.check(event)
3636
end
3737
end
38-
39-
def process_record(record)
40-
if record.url
41-
code = get_bad_response(record.url)
42-
if code
43-
puts " #{code} - #{record.class.name} #{record.id}: #{record.url}"
44-
if record.link_monitor
45-
record.link_monitor.fail!(code)
46-
else
47-
record.create_link_monitor(url: record.url, code: code)
48-
end
49-
else
50-
if record.link_monitor
51-
record.link_monitor.success!
52-
end
53-
end
54-
end
55-
56-
record.external_resources.each do |res|
57-
next unless res.url
58-
59-
code = get_bad_response(res.url)
60-
61-
if code
62-
puts " #{code} - ExternalResource #{res.id}: #{res.url}"
63-
if res.link_monitor
64-
res.link_monitor.fail!(code)
65-
else
66-
res.create_link_monitor(url: res.url, code: code)
67-
end
68-
else
69-
if res.link_monitor
70-
res.link_monitor.success!
71-
end
72-
end
73-
end
74-
end
75-
76-
# The fake return codes on an exception are so the LinkMonitor object has something
77-
# to store as "code" which might be tracked back to a particular problem.
78-
def get_bad_response(url)
79-
begin
80-
host = URI.parse(url).host rescue nil
81-
if @prev_host == host
82-
n = rand(4) + 1
83-
sleep(n)
84-
end
85-
@prev_host = host
86-
response = HTTParty.head(url, verify: false)
87-
return nil if response.code >= 200 && response.code < 400 # Success or redirects are OK
88-
return response.code
89-
rescue EOFError => e
90-
puts " #{e.class.name}: #{e}"
91-
return 490
92-
rescue SocketError => e
93-
puts " #{e.class.name}: #{e}"
94-
return 491
95-
rescue Net::ReadTimeout => e
96-
puts " #{e.class.name}: #{e}"
97-
return 492
98-
rescue StandardError => e
99-
puts " #{e.class.name}: #{e}"
100-
return 493
101-
end
102-
end
103-

‎test/unit/link_checker_test.rb‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
require 'test_helper'
2+
3+
class LinkCheckerTest < ActiveSupport::TestCase
4+
5+
test 'check passing event url' do
6+
event = events(:two)
7+
WebMock.stub_request(:head, event.url).to_return(status: 200)
8+
assert_nil event.link_monitor
9+
10+
assert_no_difference('LinkMonitor.count') do
11+
LinkChecker.check(event, log: false)
12+
end
13+
14+
assert_nil event.link_monitor
15+
end
16+
17+
test 'check failing event url' do
18+
event = events(:two)
19+
WebMock.stub_request(:head, event.url).to_return(status: 404)
20+
assert_nil event.link_monitor
21+
22+
assert_difference('LinkMonitor.count', 1) do
23+
LinkChecker.check(event, log: false)
24+
end
25+
26+
assert event.link_monitor
27+
assert event.link_monitor.failed_at
28+
assert_equal 404, event.link_monitor.code
29+
end
30+
31+
test 'check failing event url that only responds to get' do
32+
event = events(:two)
33+
WebMock.stub_request(:head, event.url).to_return(status: 405)
34+
WebMock.stub_request(:get, event.url).to_return(status: 200)
35+
assert_nil event.link_monitor
36+
37+
assert_no_difference('LinkMonitor.count') do
38+
LinkChecker.check(event, log: false)
39+
end
40+
41+
assert_nil event.link_monitor
42+
end
43+
44+
test 'follows redirect' do
45+
event = events(:two)
46+
WebMock.stub_request(:head, event.url).to_return(status: 405)
47+
WebMock.stub_request(:get, event.url).to_return(status: 302, headers: { location: 'http://website.com' })
48+
WebMock.stub_request(:get, 'http://website.com').to_return(status: 404)
49+
assert_nil event.link_monitor
50+
51+
assert_difference('LinkMonitor.count', 1) do
52+
LinkChecker.check(event, log: false)
53+
end
54+
55+
assert event.link_monitor
56+
assert event.link_monitor.failed_at
57+
assert_equal 404, event.link_monitor.code
58+
end
59+
60+
test 'avoids redirect loop' do
61+
event = events(:two)
62+
WebMock.stub_request(:head, event.url).to_return(status: 405)
63+
WebMock.stub_request(:get, event.url).to_return(status: 302, headers: { location: 'http://website.com' })
64+
WebMock.stub_request(:get, 'http://website.com').to_return(status: 302, headers: { location: event.url })
65+
assert_nil event.link_monitor
66+
67+
assert_difference('LinkMonitor.count', 1) do
68+
LinkChecker.check(event, log: false)
69+
end
70+
71+
assert event.link_monitor
72+
assert event.link_monitor.failed_at
73+
assert_equal 493, event.link_monitor.code
74+
end
75+
76+
test 'check material with external resources' do
77+
WebMock.stub_request(:head, 'http://myurl.com/123').to_return(status: 200)
78+
WebMock.stub_request(:head, 'https://tess.elixir-uk.org/').to_return(status: 200)
79+
WebMock.stub_request(:head, 'https://bio.tools/tool/SR-Tesseler').to_return(status: 404)
80+
WebMock.stub_request(:head, 'https://fairsharing.org/bsg-p123456').to_return(status: 404)
81+
material = materials(:material_with_external_resource)
82+
83+
assert_difference('LinkMonitor.count', 2) do
84+
LinkChecker.check(material, log: false)
85+
end
86+
end
87+
88+
end

0 commit comments

Comments
 (0)