diff --git a/Gemfile.lock b/Gemfile.lock index f8ca898e9..9f655bea1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -230,9 +230,12 @@ GEM concurrent-ruby (~> 1.0) i18n_data (0.17.1) simple_po_parser (~> 1.1) - icalendar (2.9.0) + icalendar (2.12.2) + base64 ice_cube (~> 0.16) - ice_cube (0.16.4) + logger + ostruct + ice_cube (0.17.0) io-console (0.8.2) irb (1.15.2) pp (>= 0.6.0) diff --git a/lib/ingestors/bioschemas_ingestor.rb b/lib/ingestors/bioschemas_ingestor.rb index fec566f22..a11793695 100644 --- a/lib/ingestors/bioschemas_ingestor.rb +++ b/lib/ingestors/bioschemas_ingestor.rb @@ -3,6 +3,8 @@ module Ingestors class BioschemasIngestor < Ingestor + include Ingestors::Concerns::SitemapHelpers + DUMMY_URL = 'https://example.com' attr_reader :verbose @@ -16,27 +18,8 @@ def self.config end def read(source_url) - sitemap_regex = nil @verbose = false - sources = if source_url.downcase.match?(/sitemap(.*)?.xml\Z/) - sitemap_message = "Parsing .xml sitemap: #{source_url}\n" - urls = SitemapParser.new(source_url, { - recurse: true, - url_regex: sitemap_regex, - headers: { 'User-Agent' => config[:user_agent] } - }).to_a.uniq.map(&:strip) - sitemap_message << "\n - #{urls.count} URLs found" - @messages << sitemap_message - urls - elsif source_url.downcase.match?(/sitemap(.*)?.txt\Z/) - sitemap_message = "Parsing .txt sitemap: #{source_url}\n" - urls = open_url(source_url).to_a.uniq.map(&:strip) - sitemap_message << "\n - #{urls.count} URLs found" - @messages << sitemap_message - urls - else - [source_url] - end + sources = parse_sitemap(source_url) provider_events = [] provider_materials = [] diff --git a/lib/ingestors/concerns/sitemap_helpers.rb b/lib/ingestors/concerns/sitemap_helpers.rb index 2f27a82f7..e9c6846c6 100644 --- a/lib/ingestors/concerns/sitemap_helpers.rb +++ b/lib/ingestors/concerns/sitemap_helpers.rb @@ -7,7 +7,7 @@ module SitemapHelpers private # Reads either a sitemap.{xml|txt} or a single URL - # Returns a list of URLs from 1 to n URLs + # Returns a list of URLs from 0 to n URLs def parse_sitemap(source_url) case source_url.downcase when /sitemap(.*)?\.xml\Z/ @@ -28,9 +28,9 @@ def parse_xml_sitemap(url) log_sitemap('xml', url, urls.count) urls - rescue StandardError => e + rescue RuntimeError => e # sitemap-parser gem raises RuntimeErrors @messages << "Extract from sitemap[#{url}] failed with: #{e.message}" - nil + [] end def parse_txt_sitemap(url) diff --git a/lib/ingestors/event_ingestion.rb b/lib/ingestors/event_ingestion.rb index 90471742d..58985bf8b 100644 --- a/lib/ingestors/event_ingestion.rb +++ b/lib/ingestors/event_ingestion.rb @@ -29,10 +29,6 @@ def convert_event_types(input) EventTypeDictionary.instance.lookup_value(input, 'title') end - def convert_location(input) - input - end - def parse_dates(input, timezone = nil) Time.use_zone(timezone) do # try to split on obvious interval markers diff --git a/lib/ingestors/ical_ingestor.rb b/lib/ingestors/ical_ingestor.rb index e91fdecfa..8df828a4b 100644 --- a/lib/ingestors/ical_ingestor.rb +++ b/lib/ingestors/ical_ingestor.rb @@ -5,6 +5,8 @@ module Ingestors class IcalIngestor < Ingestor + include Ingestors::Concerns::SitemapHelpers + def self.config { key: 'ical', @@ -14,111 +16,63 @@ def self.config end def read(url) - unless url.nil? - if url.to_s.downcase.end_with? 'sitemap.xml' - process_sitemap url - else - process_icalendar url - end + sources = parse_sitemap(url) + sources.each do |source| + process_icalendar(source) end end private - def process_sitemap(url) - # find urls for individual icalendar files - begin - sitemap = Nokogiri::XML.parse(open_url(url, raise: true)) - locs = sitemap.xpath('/ns:urlset/ns:url/ns:loc', { - 'ns' => 'http://www.sitemaps.org/schemas/sitemap/0.9' - }) - locs.each do |loc| - process_icalendar(loc.text) - end - rescue Exception => e - @messages << "Extract from sitemap[#{url}] failed with: #{e.message}" - end - - # finished - nil - end - def process_icalendar(url) # process individual ics file query = '?ical=true' - begin - # append query (if required) - file_url = url - file_url << query unless url.to_s.downcase.ends_with? query + # append query (if required) + file_url = url + file_url << query unless url.to_s.downcase.ends_with? query - # process file - events = Icalendar::Event.parse(open_url(file_url, raise: true).set_encoding('utf-8')) + # process file + data = open_url(file_url) + if data + events = Icalendar::Event.parse(data.set_encoding('utf-8')) # process each event - events.each do |e| - process_event(e) + events.each do |ical_event| + add_event(process_event(ical_event)) end - rescue Exception => e - @messages << "Process file url[#{file_url}] failed with: #{e.message}" end - # finished - nil end def process_event(calevent) - # puts "calevent: #{calevent.inspect}" - begin - # set fields - event = OpenStruct.new - event.url = calevent.url.to_s - event.title = calevent.summary.to_s - event.description = process_description calevent.description - - # puts "\n\ncalevent.description = #{calevent.description}" - # puts "\n\n... converted = #{event.description}" - - event.end = calevent.dtend&.to_time - unless calevent.dtstart.nil? - dtstart = calevent.dtstart - event.start = dtstart&.to_time - tzid = dtstart.ical_params['tzid'] - event.timezone = tzid.first.to_s if !tzid.nil? and tzid.size > 0 - end + # set fields + event = OpenStruct.new + event.url = calevent.url&.to_s + event.title = calevent.summary&.to_s + event.description = process_description(calevent.description) + + event.end = calevent.dtend&.to_time + unless calevent.dtstart.nil? + dtstart = calevent.dtstart + event.start = dtstart&.to_time + tzid = dtstart.ical_params['tzid'] + event.timezone = tzid.first.to_s if tzid.present? + end + if calevent.location event.venue = calevent.location.to_s if calevent.location.downcase.include?('online') event.online = true event.city = nil event.postcode = nil event.country = nil - else - location = convert_location(calevent.location) - event.city = location['suburb'] unless location['suburb'].nil? - event.country = location['country'] unless location['country'].nil? - event.postcode = location['postcode'] unless location['postcode'].nil? end - event.keywords = [] - unless calevent.categories.nil? or calevent.categories.first.nil? - cats = calevent.categories.first - if cats.is_a?(Icalendar::Values::Array) - cats.each do |item| - event.keywords << item.to_s.lstrip - end - else - event.keywords << cats.to_s.strip - end - end - - # store event - @events << event - rescue Exception => e - @messages << "Process iCalendar failed with: #{e.message}" end - # finished - nil + event.keywords = calevent.categories.flatten.map(&:strip) + + event end def process_description(input) diff --git a/lib/ingestors/indico_ingestor.rb b/lib/ingestors/indico_ingestor.rb index 4b8f0f5d4..5165f0d96 100644 --- a/lib/ingestors/indico_ingestor.rb +++ b/lib/ingestors/indico_ingestor.rb @@ -22,7 +22,6 @@ def read(source_url) @token = Rails.application.config.secrets.indico_api_token @verbose = false sources = parse_sitemap(source_url) - return if sources.nil? sources.each do |url| process_url(url) diff --git a/test/test_helper.rb b/test/test_helper.rb index df2d62de6..17218b588 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -48,6 +48,7 @@ class ActiveSupport::TestCase teardown do User.current_user = nil + Space.current_space = nil end # WARNING: Do not be tempted to include Devise TestHelpers here (e.g. include Devise::TestHelpers) diff --git a/test/unit/ingestors/ical_ingestor_test.rb b/test/unit/ingestors/ical_ingestor_test.rb index cb6ff5a54..c4cd7754f 100644 --- a/test/unit/ingestors/ical_ingestor_test.rb +++ b/test/unit/ingestors/ical_ingestor_test.rb @@ -26,7 +26,7 @@ class IcalIngestorTest < ActiveSupport::TestCase assert ingestor.events.empty? assert ingestor.materials.empty? - assert_includes ingestor.messages, 'Extract from sitemap[https://missing.org/sitemap.xml] failed with: 404 ' + assert_includes ingestor.messages, 'Extract from sitemap[https://missing.org/sitemap.xml] failed with: HTTP request to https://missing.org/sitemap.xml failed' end test 'ingest valid sitemap' do @@ -68,7 +68,7 @@ class IcalIngestorTest < ActiveSupport::TestCase # check individual events # check not found - assert_includes ingestor.messages, "Process file url\[https://pawsey.org.au/events/\?ical=true\] failed with: 404 " + assert_includes ingestor.messages, "Couldn't open URL https://pawsey.org.au/events/?ical=true: 404 " # check rejected event = ingestor.events.detect { |e| e.title == 'NVIDIA cuQuantum Session' }