Use PrivateAddressCheck in ingestion

fbacall · fbacall · commit 9cb068329c3a · 2026-04-07T18:20:12.000+01:00
diff --git a/lib/ingestors/bioschemas_ingestor.rb b/lib/ingestors/bioschemas_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'tess_rdf_extractors'
 
 module Ingestors
diff --git a/lib/ingestors/carpentries_ingestor.rb b/lib/ingestors/carpentries_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 
@@ -48,7 +47,7 @@ def process_workshop(workshop)
 
     def curricula
       Rails.cache.fetch('carpentries-curricula', expires_in: 1.week) do
-        json = URI.open(CURRICULA_URL).read
+        json = open_url(CURRICULA_URL).read
         hash = {}
         JSON.parse(json).each do |c|
           hash[c['slug']] = c
diff --git a/lib/ingestors/event_csv_ingestor.rb b/lib/ingestors/event_csv_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 
 module Ingestors
diff --git a/lib/ingestors/eventbrite_ingestor.rb b/lib/ingestors/eventbrite_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 
 module Ingestors
diff --git a/lib/ingestors/github_ingestor.rb b/lib/ingestors/github_ingestor.rb
@@ -1,6 +1,4 @@
 # frozen_string_literal: true
-
-require 'open-uri'
 require 'json'
 require 'httparty'
 require 'nokogiri'
diff --git a/lib/ingestors/heptraining/gray_scott_ingestor.rb b/lib/ingestors/heptraining/gray_scott_ingestor.rb
@@ -1,6 +1,5 @@
 require 'icalendar'
 require 'nokogiri'
-require 'open-uri'
 require 'tzinfo'
 
 module Ingestors
diff --git a/lib/ingestors/ical_ingestor.rb b/lib/ingestors/ical_ingestor.rb
@@ -1,6 +1,5 @@
 require 'icalendar'
 require 'nokogiri'
-require 'open-uri'
 require 'tzinfo'
 
 module Ingestors
diff --git a/lib/ingestors/indico_ingestor.rb b/lib/ingestors/indico_ingestor.rb
@@ -2,7 +2,6 @@
 
 require 'icalendar'
 require 'nokogiri'
-require 'open-uri'
 require 'tzinfo'
 
 module Ingestors
diff --git a/lib/ingestors/ingestor.rb b/lib/ingestors/ingestor.rb
@@ -1,4 +1,6 @@
 require 'sitemap-parser'
+require 'private_address_check'
+require 'private_address_check/tcpsocket_ext'
 
 module Ingestors
   class Ingestor
@@ -71,7 +73,14 @@ def open_url(url, raise: false, token: nil)
       redirect_attempts = 5
       options['Authorization'] = "Bearer #{token}" unless token.nil?
       begin
-        URI(url).open(options)
+        PrivateAddressCheck.only_public_connections do
+          URI(url).open(options)
+        end
+      rescue PrivateAddressCheck::PrivateConnectionAttemptedError
+        raise "Couldn't open URL #{url}" if raise
+        @messages << "Couldn't open URL #{url}"
+
+        nil
       rescue OpenURI::HTTPRedirect => e
         url = e.uri.to_s
         retry if (redirect_attempts -= 1) > 0
diff --git a/lib/ingestors/libcal_ingestor.rb b/lib/ingestors/libcal_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 
 module Ingestors
diff --git a/lib/ingestors/llm_ingestor.rb b/lib/ingestors/llm_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 
diff --git a/lib/ingestors/material_csv_ingestor.rb b/lib/ingestors/material_csv_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 require 'cgi'
 
diff --git a/lib/ingestors/oscm_ingestor.rb b/lib/ingestors/oscm_ingestor.rb
@@ -1,5 +1,4 @@
 require 'icalendar'
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 require 'openssl'
diff --git a/lib/ingestors/taxila/citizen_science_ingestor.rb b/lib/ingestors/taxila/citizen_science_ingestor.rb
@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 
diff --git a/lib/ingestors/taxila/dans_ingestor.rb b/lib/ingestors/taxila/dans_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 
diff --git a/lib/ingestors/taxila/dcc_ingestor.rb b/lib/ingestors/taxila/dcc_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 
diff --git a/lib/ingestors/taxila/dtls_ingestor.rb b/lib/ingestors/taxila/dtls_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 
diff --git a/lib/ingestors/taxila/fourtu_llm_ingestor.rb b/lib/ingestors/taxila/fourtu_llm_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 
diff --git a/lib/ingestors/taxila/han_ingestor.rb b/lib/ingestors/taxila/han_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 
diff --git a/lib/ingestors/taxila/lcrdm_ingestor.rb b/lib/ingestors/taxila/lcrdm_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 
diff --git a/lib/ingestors/taxila/leiden_ingestor.rb b/lib/ingestors/taxila/leiden_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 
diff --git a/lib/ingestors/taxila/maastricht_ingestor.rb b/lib/ingestors/taxila/maastricht_ingestor.rb
@@ -1,5 +1,4 @@
 require 'icalendar'
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 
diff --git a/lib/ingestors/taxila/nwo_ingestor.rb b/lib/ingestors/taxila/nwo_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 
diff --git a/lib/ingestors/taxila/odissei_ingestor.rb b/lib/ingestors/taxila/odissei_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 
diff --git a/lib/ingestors/taxila/oscd_ingestor.rb b/lib/ingestors/taxila/oscd_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 
diff --git a/lib/ingestors/taxila/osci_ingestor.rb b/lib/ingestors/taxila/osci_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 
diff --git a/lib/ingestors/taxila/oscm_ingestor.rb b/lib/ingestors/taxila/oscm_ingestor.rb
@@ -1,5 +1,4 @@
 require 'icalendar'
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 require 'openssl'
diff --git a/lib/ingestors/taxila/rdnl_ingestor.rb b/lib/ingestors/taxila/rdnl_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 
diff --git a/lib/ingestors/taxila/rst_ingestor.rb b/lib/ingestors/taxila/rst_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 
diff --git a/lib/ingestors/taxila/rug_ingestor.rb b/lib/ingestors/taxila/rug_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 
diff --git a/lib/ingestors/taxila/sense_ingestor.rb b/lib/ingestors/taxila/sense_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 
diff --git a/lib/ingestors/taxila/surf_ingestor.rb b/lib/ingestors/taxila/surf_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 require 'active_support/core_ext/hash'
diff --git a/lib/ingestors/taxila/tdcc_ingestor.rb b/lib/ingestors/taxila/tdcc_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 
diff --git a/lib/ingestors/taxila/uhasselt_ingestor.rb b/lib/ingestors/taxila/uhasselt_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 
diff --git a/lib/ingestors/taxila/utwente_ingestor.rb b/lib/ingestors/taxila/utwente_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 
 module Ingestors
diff --git a/lib/ingestors/taxila/uu_ingestor.rb b/lib/ingestors/taxila/uu_ingestor.rb
@@ -1,5 +1,4 @@
 require 'icalendar'
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 
@@ -112,7 +111,7 @@ def process_uu(url)
             # Now fetch the page to get the event date (until it is added to the RSS feed)
             # if event.url.starts_with('https://')
             # should we do more against data exfiltration? URI.open is a known hazard
-            # page = Nokogiri::XML(URI.open(event.url))
+            # page = Nokogiri::XML(open_url(event.url))
             # event.description = convert_description page.css('.content-block__inner').first.inner_html
             # end
             add_event(event)
diff --git a/lib/ingestors/taxila/uva_ingestor.rb b/lib/ingestors/taxila/uva_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 
 module Ingestors
diff --git a/lib/ingestors/taxila/vu_material_ingestor.rb b/lib/ingestors/taxila/vu_material_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 
diff --git a/lib/ingestors/taxila/wur_ingestor.rb b/lib/ingestors/taxila/wur_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 require 'nokogiri'
 
diff --git a/lib/ingestors/tess_event_ingestor.rb b/lib/ingestors/tess_event_ingestor.rb
@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'csv'
 
 module Ingestors
diff --git a/test/unit/ingestors/ingestor_test.rb b/test/unit/ingestors/ingestor_test.rb
@@ -270,30 +270,24 @@ def run_filter(source_filter)
     assert_equal 'ok', result.read
   end
 
-  test 'open_url raises HTTPRedirect after too many retries' do
+  test 'open_url raises exception after too many redirects' do
     ingestor = DummyIngestor.new
-    fake_uri = URI('https://example.com/')
+    WebMock.stub_request(:any, 'https://redirect.loop/').to_return(status: 301,
+                                                                   headers: { 'Location' => 'https://redirect.loop/' })
 
-    URI.stub(:parse, fake_uri) do
-      fake_uri.define_singleton_method(:open) do |*_args|
-        raise OpenURI::HTTPRedirect.new('Redirect', 1, URI('https://redirected.com'))
-      end
-
-      assert_raises(OpenURI::HTTPRedirect) do
-        ingestor.open_url('https://example.com/')
-      end
+    assert_raises(OpenURI::HTTPRedirect) do
+      ingestor.open_url('https://redirect.loop/')
     end
   end
 
-  test 'open_url raises HTTPError' do
+  test 'open_url logs error status' do
     ingestor = DummyIngestor.new
-    stub_request(:get, 'https://bad.com')
-      .to_raise(OpenURI::HTTPError.new('404 not found', StringIO.new))
+    WebMock.stub_request(:any, 'https://bad.com/').to_return(status: 404)
 
     result = ingestor.open_url('https://bad.com')
     assert_nil result
     assert_includes ingestor.instance_variable_get(:@messages).last,
-                    "Couldn't open URL https://bad.com: 404 not found"
+                    "Couldn't open URL https://bad.com: 404"
   end
 
   test 'get_redirected_url follows meta refresh redirect' do
@@ -360,4 +354,36 @@ def run_filter(source_filter)
     assert_equal space, event.space
   end
 
+  test 'handles HTTP errors and SSRF attempts' do
+    WebMock.stub_request(:any, 'http://200host.com').to_return(status: 200, body: 'hi')
+    WebMock.stub_request(:any, 'http://404host.com').to_return(status: 404, body: 'hi')
+    WebMock.stub_request(:any, 'http://slowhost.com').to_timeout
+
+    ingestor = Ingestors::BioschemasIngestor.new
+
+    # 200
+    assert ingestor.open_url('http://200host.com')
+
+    # 404
+    assert_nil ingestor.open_url('http://404host.com')
+    assert_equal "Couldn't open URL http://404host.com: 404 ", ingestor.messages.last
+    assert_raises(OpenURI::HTTPError) do
+      ingestor.open_url('http://404host.com', raise: true)
+    end
+
+    # private address
+    with_net_connection do # Allow request through to be caught by private_address_check
+      assert_nil ingestor.open_url('http://192.168.0.1')
+      assert_equal "Couldn't open URL http://192.168.0.1", ingestor.messages.last
+      assert_raises(RuntimeError) do
+        ingestor.open_url('http://192.168.0.1', raise: true)
+      end
+    end
+
+    # timeout
+    assert_raises(Net::OpenTimeout) do
+      ingestor.open_url('http://slowhost.com') # Always raises
+    end
+  end
+
 end

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,3 @@`
`1`		`-require 'open-uri'`
`2`	`1`	`require 'tess_rdf_extractors'`
`3`	`2`
`4`	`3`	`module Ingestors`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,3 @@`
`1`		`-require 'open-uri'`
`2`	`1`	`require 'csv'`
`3`	`2`
`4`	`3`	`module Ingestors`