Merge pull request #1274 from ElixirTeSS/ical-bump

icalendar bump + fixes

Author: fbacall

Gemfile.lock

@@ -230,9 +230,12 @@ GEM
       concurrent-ruby (~> 1.0)
     i18n_data (0.17.1)
       simple_po_parser (~> 1.1)
-    icalendar (2.9.0)
+    icalendar (2.12.2)
+      base64
       ice_cube (~> 0.16)
-    ice_cube (0.16.4)
+      logger
+      ostruct
+    ice_cube (0.17.0)
     io-console (0.8.2)
     irb (1.15.2)
       pp (>= 0.6.0)

lib/ingestors/bioschemas_ingestor.rb

@@ -2,6 +2,8 @@
 
 module Ingestors
   class BioschemasIngestor < Ingestor
+    include Ingestors::Concerns::SitemapHelpers
+
     DUMMY_URL = 'https://example.com'
 
     attr_reader :verbose
@@ -15,27 +17,8 @@ def self.config
     end
 
     def read(source_url)
-      sitemap_regex = nil
       @verbose = false
-      sources = if source_url.downcase.match?(/sitemap(.*)?.xml\Z/)
-                  sitemap_message = "Parsing .xml sitemap: #{source_url}\n"
-                  urls = SitemapParser.new(source_url, {
-                    recurse: true,
-                    url_regex: sitemap_regex,
-                    headers: { 'User-Agent' => config[:user_agent] }
-                  }).to_a.uniq.map(&:strip)
-                  sitemap_message << "\n - #{urls.count} URLs found"
-                  @messages << sitemap_message
-                  urls
-                elsif source_url.downcase.match?(/sitemap(.*)?.txt\Z/)
-                  sitemap_message = "Parsing .txt sitemap: #{source_url}\n"
-                  urls = open_url(source_url).to_a.uniq.map(&:strip)
-                  sitemap_message << "\n - #{urls.count} URLs found"
-                  @messages << sitemap_message
-                  urls
-                else
-                  [source_url]
-                end
+      sources = parse_sitemap(source_url)
 
       provider_events = []
       provider_materials = []

lib/ingestors/concerns/sitemap_helpers.rb

@@ -7,7 +7,7 @@ module SitemapHelpers
       private
 
       # Reads either a sitemap.{xml|txt} or a single URL
-      # Returns a list of URLs from 1 to n URLs
+      # Returns a list of URLs from 0 to n URLs
       def parse_sitemap(source_url)
         case source_url.downcase
         when /sitemap(.*)?\.xml\Z/
@@ -28,9 +28,9 @@ def parse_xml_sitemap(url)
 
         log_sitemap('xml', url, urls.count)
         urls
-      rescue StandardError => e
+      rescue RuntimeError => e # sitemap-parser gem raises RuntimeErrors
         @messages << "Extract from sitemap[#{url}] failed with: #{e.message}"
-        nil
+        []
       end
 
       def parse_txt_sitemap(url)

lib/ingestors/event_ingestion.rb

@@ -29,10 +29,6 @@ def convert_event_types(input)
       EventTypeDictionary.instance.lookup_value(input, 'title')
     end
 
-    def convert_location(input)
-      input
-    end
-
     def parse_dates(input, timezone = nil)
       Time.use_zone(timezone) do
         # try to split on obvious interval markers

lib/ingestors/ical_ingestor.rb

@@ -4,6 +4,8 @@
 
 module Ingestors
   class IcalIngestor < Ingestor
+    include Ingestors::Concerns::SitemapHelpers
+
     def self.config
       {
         key: 'ical',
@@ -13,111 +15,63 @@ def self.config
     end
 
     def read(url)
-      unless url.nil?
-        if url.to_s.downcase.end_with? 'sitemap.xml'
-          process_sitemap url
-        else
-          process_icalendar url
-        end
+      sources = parse_sitemap(url)
+      sources.each do |source|
+        process_icalendar(source)
       end
     end
 
     private
 
-    def process_sitemap(url)
-      # find urls for individual icalendar files
-      begin
-        sitemap = Nokogiri::XML.parse(open_url(url, raise: true))
-        locs = sitemap.xpath('/ns:urlset/ns:url/ns:loc', {
-                               'ns' => 'http://www.sitemaps.org/schemas/sitemap/0.9'
-                             })
-        locs.each do |loc|
-          process_icalendar(loc.text)
-        end
-      rescue Exception => e
-        @messages << "Extract from sitemap[#{url}] failed with: #{e.message}"
-      end
-
-      # finished
-      nil
-    end
-
     def process_icalendar(url)
       # process individual ics file
       query = '?ical=true'
 
-      begin
-        # append query  (if required)
-        file_url = url
-        file_url << query unless url.to_s.downcase.ends_with? query
+      # append query  (if required)
+      file_url = url
+      file_url << query unless url.to_s.downcase.ends_with? query
 
-        # process file
-        events = Icalendar::Event.parse(open_url(file_url, raise: true).set_encoding('utf-8'))
+      # process file
+      data = open_url(file_url)
+      if data
+        events = Icalendar::Event.parse(data.set_encoding('utf-8'))
 
         # process each event
-        events.each do |e|
-          process_event(e)
+        events.each do |ical_event|
+          add_event(process_event(ical_event))
         end
-      rescue Exception => e
-        @messages << "Process file url[#{file_url}] failed with: #{e.message}"
       end
-
       # finished
-      nil
     end
 
     def process_event(calevent)
-      # puts "calevent: #{calevent.inspect}"
-      begin
-        # set fields
-        event = OpenStruct.new
-        event.url = calevent.url.to_s
-        event.title = calevent.summary.to_s
-        event.description = process_description calevent.description
-
-        # puts "\n\ncalevent.description = #{calevent.description}"
-        # puts "\n\n...        converted = #{event.description}"
-
-        event.end = calevent.dtend&.to_time
-        unless calevent.dtstart.nil?
-          dtstart = calevent.dtstart
-          event.start = dtstart&.to_time
-          tzid = dtstart.ical_params['tzid']
-          event.timezone = tzid.first.to_s if !tzid.nil? and tzid.size > 0
-        end
+      # set fields
+      event = OpenStruct.new
+      event.url = calevent.url&.to_s
+      event.title = calevent.summary&.to_s
+      event.description = process_description(calevent.description)
+
+      event.end = calevent.dtend&.to_time
+      unless calevent.dtstart.nil?
+        dtstart = calevent.dtstart
+        event.start = dtstart&.to_time
+        tzid = dtstart.ical_params['tzid']
+        event.timezone = tzid.first.to_s if tzid.present?
+      end
 
+      if calevent.location
         event.venue = calevent.location.to_s
         if calevent.location.downcase.include?('online')
           event.online = true
           event.city = nil
           event.postcode = nil
           event.country = nil
-        else
-          location = convert_location(calevent.location)
-          event.city = location['suburb'] unless location['suburb'].nil?
-          event.country = location['country'] unless location['country'].nil?
-          event.postcode = location['postcode'] unless location['postcode'].nil?
         end
-        event.keywords = []
-        unless calevent.categories.nil? or calevent.categories.first.nil?
-          cats = calevent.categories.first
-          if cats.is_a?(Icalendar::Values::Array)
-            cats.each do |item|
-              event.keywords << item.to_s.lstrip
-            end
-          else
-            event.keywords << cats.to_s.strip
-          end
-        end
-
-        # store event
-        @events << event
-      rescue Exception => e
-        @messages << "Process iCalendar failed with: #{e.message}"
       end
 
-      # finished
-      nil
+      event.keywords = calevent.categories.flatten.map(&:strip)
+
+      event
     end
 
     def process_description(input)

lib/ingestors/indico_ingestor.rb

@@ -21,7 +21,6 @@ def read(source_url)
       @token = Rails.application.config.secrets.indico_api_token
       @verbose = false
       sources = parse_sitemap(source_url)
-      return if sources.nil?
 
       sources.each do |url|
         process_url(url)

test/test_helper.rb

@@ -48,6 +48,7 @@ class ActiveSupport::TestCase
 
   teardown do
     User.current_user = nil
+    Space.current_space = nil
   end
 
   # WARNING: Do not be tempted to include Devise TestHelpers here (e.g. include Devise::TestHelpers)

test/unit/ingestors/ical_ingestor_test.rb

@@ -26,7 +26,7 @@ class IcalIngestorTest < ActiveSupport::TestCase
 
     assert ingestor.events.empty?
     assert ingestor.materials.empty?
-    assert_includes ingestor.messages, 'Extract from sitemap[https://missing.org/sitemap.xml] failed with: 404 '
+    assert_includes ingestor.messages, 'Extract from sitemap[https://missing.org/sitemap.xml] failed with: HTTP request to https://missing.org/sitemap.xml failed'
   end
 
   test 'ingest valid sitemap' do
@@ -68,7 +68,7 @@ class IcalIngestorTest < ActiveSupport::TestCase
 
     # check individual events
     # check not found
-    assert_includes ingestor.messages, "Process file url\[https://pawsey.org.au/events/\?ical=true\] failed with: 404 "
+    assert_includes ingestor.messages, "Couldn't open URL https://pawsey.org.au/events/?ical=true: 404 "
 
     # check rejected
     event = ingestor.events.detect { |e| e.title == 'NVIDIA cuQuantum Session' }