Skip to content

Commit 7ab91a0

Browse files
fbacallfbacall
authored
Merge pull request #1164 from ElixirTeSS/link-check-get
GET fallback for link checker
2 parents 81930b8 + 9777792 commit 7ab91a0

3 files changed

Lines changed: 215 additions & 82 deletions

File tree

‎lib/link_checker.rb‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
class LinkChecker
2+
TIMEOUT = 5
3+
attr_reader :log
4+
5+
def initialize(log: true)
6+
@log = log
7+
@cache = {}
8+
end
9+
10+
def check(collection)
11+
collection.find_each do |record|
12+
check_record(record)
13+
end
14+
end
15+
16+
def check_record(record)
17+
[record, *record.external_resources].each do |item|
18+
next if item.url.blank?
19+
code = @cache.fetch(item.url) { bad_response(item.url) } # Cache the result, there could be multiple resources linked to same URL
20+
if code
21+
puts " #{code} - #{item.class.name} #{item.id}: #{item.url}" if log
22+
if item.link_monitor
23+
item.link_monitor.fail!(code)
24+
else
25+
item.create_link_monitor(url: item.url, code: code)
26+
end
27+
else
28+
if item.link_monitor
29+
item.link_monitor.success!
30+
end
31+
end
32+
end
33+
end
34+
35+
private
36+
37+
# The fake return codes on an exception are so the LinkMonitor object has something
38+
# to store as "code" which might be tracked back to a particular problem.
39+
def bad_response(url)
40+
begin
41+
host = URI.parse(url).host rescue nil
42+
if @prev_host == host
43+
n = rand(4) + 1 # Add some delay between requests to the same host to prevent flooding
44+
sleep(n)
45+
end
46+
@prev_host = host
47+
code = HTTParty.head(url, verify: false, open_timeout: TIMEOUT, read_timeout: TIMEOUT).code
48+
code = get_code(url) if code == 400 || code == 405 # Try a GET if HEAD not allowed (or generic 400 error)
49+
return nil if code >= 200 && code < 400 # Success or redirects are OK
50+
return code
51+
rescue EOFError => e
52+
puts " #{e.class.name}: #{e}" if log
53+
return 490
54+
rescue SocketError => e
55+
puts " #{e.class.name}: #{e}" if log
56+
return 491
57+
rescue Timeout::Error => e
58+
puts " #{e.class.name}: #{e}" if log
59+
return 492
60+
rescue StandardError => e
61+
puts " #{e.class.name}: #{e}" if log
62+
return 493
63+
end
64+
end
65+
66+
# Gets a response code using a GET request, for the case where HEAD is not supported.
67+
def get_code(url, redirect_limit: 5, open_timeout: TIMEOUT, read_timeout: TIMEOUT, use_range: true)
68+
raise StandardError, 'too many redirects' if redirect_limit <= 0
69+
70+
uri = URI(url)
71+
72+
# Prepare the HTTP connection
73+
http = Net::HTTP.new(uri.host, uri.port)
74+
http.use_ssl = (uri.scheme == 'https')
75+
http.open_timeout = open_timeout
76+
http.read_timeout = read_timeout
77+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
78+
79+
# Build GET request (not HEAD)
80+
req = Net::HTTP::Get.new(uri)
81+
req['Range'] = 'bytes=0-0' if use_range # fetch just the first byte if supported
82+
83+
http.start do |connection|
84+
connection.request(req) do |res|
85+
case res
86+
when Net::HTTPRedirection
87+
location = res['location']
88+
new_uri = URI.join(uri, location).to_s
89+
return get_code(new_uri,
90+
redirect_limit: redirect_limit - 1,
91+
open_timeout: open_timeout,
92+
read_timeout: read_timeout,
93+
use_range: use_range)
94+
else
95+
code = res.code.to_i
96+
# Do not call res.read_body — we don't want to download the content
97+
return code
98+
end
99+
end
100+
end
101+
end
102+
end

‎lib/tasks/check_urls.rake‎

Lines changed: 9 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -7,97 +7,24 @@ namespace :tess do
77

88
desc 'Check material URLs for dead links'
99
task check_material_urls: :environment do
10-
check_materials
10+
puts 'Checking material URLs'
11+
LinkChecker.new.check(Material)
1112
end
1213

1314
desc 'Check event URLs for dead links'
1415
task check_event_urls: :environment do
15-
check_events
16+
puts 'Checking event URLs'
17+
LinkChecker.new.check(Event)
1618
end
1719

1820
desc 'Check event and material URLs for dead links'
1921
task check_resource_urls: :environment do
20-
check_materials
21-
check_events
22-
end
23-
end
22+
lc = LinkChecker.new
2423

25-
def check_materials
26-
puts 'Checking material URLs'
27-
Material.find_each do |mat|
28-
process_record(mat)
29-
end
30-
end
24+
puts 'Checking material URLs'
25+
lc.check(Material)
3126

32-
def check_events
33-
puts 'Checking event URLs'
34-
Event.find_each do |event|
35-
process_record(event)
27+
puts 'Checking event URLs'
28+
lc.check(Event)
3629
end
3730
end
38-
39-
def process_record(record)
40-
if record.url
41-
code = get_bad_response(record.url)
42-
if code
43-
puts " #{code} - #{record.class.name} #{record.id}: #{record.url}"
44-
if record.link_monitor
45-
record.link_monitor.fail!(code)
46-
else
47-
record.create_link_monitor(url: record.url, code: code)
48-
end
49-
else
50-
if record.link_monitor
51-
record.link_monitor.success!
52-
end
53-
end
54-
end
55-
56-
record.external_resources.each do |res|
57-
next unless res.url
58-
59-
code = get_bad_response(res.url)
60-
61-
if code
62-
puts " #{code} - ExternalResource #{res.id}: #{res.url}"
63-
if res.link_monitor
64-
res.link_monitor.fail!(code)
65-
else
66-
res.create_link_monitor(url: res.url, code: code)
67-
end
68-
else
69-
if res.link_monitor
70-
res.link_monitor.success!
71-
end
72-
end
73-
end
74-
end
75-
76-
# The fake return codes on an exception are so the LinkMonitor object has something
77-
# to store as "code" which might be tracked back to a particular problem.
78-
def get_bad_response(url)
79-
begin
80-
host = URI.parse(url).host rescue nil
81-
if @prev_host == host
82-
n = rand(4) + 1
83-
sleep(n)
84-
end
85-
@prev_host = host
86-
response = HTTParty.head(url, verify: false)
87-
return nil if response.code >= 200 && response.code < 400 # Success or redirects are OK
88-
return response.code
89-
rescue EOFError => e
90-
puts " #{e.class.name}: #{e}"
91-
return 490
92-
rescue SocketError => e
93-
puts " #{e.class.name}: #{e}"
94-
return 491
95-
rescue Net::ReadTimeout => e
96-
puts " #{e.class.name}: #{e}"
97-
return 492
98-
rescue StandardError => e
99-
puts " #{e.class.name}: #{e}"
100-
return 493
101-
end
102-
end
103-

‎test/unit/link_checker_test.rb‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
require 'test_helper'
2+
3+
class LinkCheckerTest < ActiveSupport::TestCase
4+
5+
setup do
6+
@link_checker = LinkChecker.new(log: false)
7+
end
8+
9+
test 'check collection' do
10+
event = events(:two)
11+
WebMock.stub_request(:head, event.url).to_return(status: 200)
12+
assert_nil event.link_monitor
13+
14+
assert_no_difference('LinkMonitor.count') do
15+
@link_checker.check(Event.where(id: event.id))
16+
end
17+
18+
assert_nil event.link_monitor
19+
end
20+
21+
test 'check passing event url' do
22+
event = events(:two)
23+
WebMock.stub_request(:head, event.url).to_return(status: 200)
24+
assert_nil event.link_monitor
25+
26+
assert_no_difference('LinkMonitor.count') do
27+
@link_checker.check_record(event)
28+
end
29+
30+
assert_nil event.link_monitor
31+
end
32+
33+
test 'check failing event url' do
34+
event = events(:two)
35+
WebMock.stub_request(:head, event.url).to_return(status: 404)
36+
assert_nil event.link_monitor
37+
38+
assert_difference('LinkMonitor.count', 1) do
39+
@link_checker.check_record(event)
40+
end
41+
42+
assert event.link_monitor
43+
assert event.link_monitor.failed_at
44+
assert_equal 404, event.link_monitor.code
45+
end
46+
47+
test 'check failing event url that only responds to get' do
48+
event = events(:two)
49+
WebMock.stub_request(:head, event.url).to_return(status: 405)
50+
WebMock.stub_request(:get, event.url).to_return(status: 200)
51+
assert_nil event.link_monitor
52+
53+
assert_no_difference('LinkMonitor.count') do
54+
@link_checker.check_record(event)
55+
end
56+
57+
assert_nil event.link_monitor
58+
end
59+
60+
test 'follows redirect' do
61+
event = events(:two)
62+
WebMock.stub_request(:head, event.url).to_return(status: 405)
63+
WebMock.stub_request(:get, event.url).to_return(status: 302, headers: { location: 'http://website.com' })
64+
WebMock.stub_request(:get, 'http://website.com').to_return(status: 404)
65+
assert_nil event.link_monitor
66+
67+
assert_difference('LinkMonitor.count', 1) do
68+
@link_checker.check_record(event)
69+
end
70+
71+
assert event.link_monitor
72+
assert event.link_monitor.failed_at
73+
assert_equal 404, event.link_monitor.code
74+
end
75+
76+
test 'avoids redirect loop' do
77+
event = events(:two)
78+
WebMock.stub_request(:head, event.url).to_return(status: 405)
79+
WebMock.stub_request(:get, event.url).to_return(status: 302, headers: { location: 'http://website.com' })
80+
WebMock.stub_request(:get, 'http://website.com').to_return(status: 302, headers: { location: event.url })
81+
assert_nil event.link_monitor
82+
83+
assert_difference('LinkMonitor.count', 1) do
84+
@link_checker.check_record(event)
85+
end
86+
87+
assert event.link_monitor
88+
assert event.link_monitor.failed_at
89+
assert_equal 493, event.link_monitor.code
90+
end
91+
92+
test 'check material with external resources' do
93+
WebMock.stub_request(:head, 'http://myurl.com/123').to_return(status: 200)
94+
WebMock.stub_request(:head, 'https://tess.elixir-uk.org/').to_return(status: 200)
95+
WebMock.stub_request(:head, 'https://bio.tools/tool/SR-Tesseler').to_return(status: 404)
96+
WebMock.stub_request(:head, 'https://fairsharing.org/bsg-p123456').to_return(status: 404)
97+
material = materials(:material_with_external_resource)
98+
99+
assert_difference('LinkMonitor.count', 2) do
100+
@link_checker.check_record(material)
101+
end
102+
end
103+
104+
end

0 commit comments

Comments
 (0)