feat(indico-ingestor): extended ical ingestor to .ics and Indico

Author: kennethrioja

lib/ingestors/concerns/ical_ingestor_export_url.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Ingestors
+  module Concerns
+    # Gets the proper URL to export an ics or ical
+    module IcalIngestorExportUrl
+      private
+
+      # 1. If the host includes 'indico', ensures the path ends with '/events.ics'.
+      # 2. If the path already ends with '/events.ics', return as-is.
+      # 3. Otherwise, append '?ical=true' query param if not already present.
+      #
+      # This method never mutates the original URL string.
+      # Returns the updated URL string or nil if input is blank.
+      def to_export(url)
+        return nil if url.blank?
+
+        uri = URI.parse(url)
+        path = uri.path.to_s
+
+        if uri.host&.include?('indico')
+          ensure_events_ics_path(uri)
+        elsif path.match?(%r{/(event|events)\.ics\z})
+          uri.to_s
+        else
+          ensure_ical_query(uri)
+        end
+      end
+
+      # Ensures the Indico URL ends with '/events.ics'
+      def ensure_events_ics_path(uri)
+        if uri.path&.include?('event')
+          uri.path = File.join(uri.path, 'event.ics') unless uri.path.end_with?('/event.ics')
+        elsif uri.path&.include?('category')
+          uri.path = File.join(uri.path, 'events.ics') unless uri.path.end_with?('/events.ics')
+        end
+        uri.to_s
+      end
+
+      # Ensures the URL has '?ical=true' in its query params
+      def ensure_ical_query(uri)
+        query = URI.decode_www_form(uri.query.to_s).to_h
+        query['ical'] = 'true' unless query['ical'] == 'true'
+        uri.query = URI.encode_www_form(query)
+        uri.to_s
+      end
+    end
+  end
+end

lib/ingestors/concerns/sitemap_helpers.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Ingestors
+  module Concerns
+    # From a sitemap.{xml|txt} or a single URL, get the list of URLs (= sources)
+    module SitemapHelpers
+      private
+
+      # Reads either a sitemap.{xml|txt} or a single URL
+      # Returns a list of URLs from 1 to n URLs
+      def get_sources(source_url)
+        case source_url.downcase
+        when /sitemap(.*)?\.xml\Z/
+          parse_xml_sitemap(source_url)
+        when /sitemap(.*)?\.txt\Z/
+          parse_txt_sitemap(source_url)
+        else
+          [source_url]
+        end
+      end
+
+      def parse_xml_sitemap(url)
+        urls = SitemapParser.new(
+          url,
+          recurse: true,
+          headers: { 'User-Agent' => config[:user_agent] }
+        ).to_a.uniq.map(&:strip)
+
+        log_sitemap('xml', url, urls.count)
+        urls
+      rescue StandardError => e
+        @messages << "Extract from sitemap[#{url}] failed with: #{e.message}"
+        nil
+      end
+
+      def parse_txt_sitemap(url)
+        urls = open_url(url).to_a.uniq.map(&:strip)
+
+        log_sitemap('txt', url, urls.count)
+        urls
+      end
+
+      def log_sitemap(ext, url, count)
+        @messages << "Parsing .#{ext} sitemap: #{url}\n - #{count} URLs found"
+      end
+    end
+  end
+end

lib/ingestors/ical_ingestor.rb

@@ -1,130 +1,125 @@
+# frozen_string_literal: true
+
 require 'icalendar'
 require 'nokogiri'
 require 'open-uri'
 require 'tzinfo'
 
 module Ingestors
+  # Reads from direct ical / .ics / Indico (event or category) URLs, .xml sitemaps, and .txt sitemaps.
   class IcalIngestor < Ingestor
+    include Ingestors::Concerns::SitemapHelpers
+    include Ingestors::Concerns::IcalIngestorExportUrl
+
     def self.config
       {
         key: 'ical',
-        title: 'iCalendar',
+        title: 'iCalendar / Indico / .ics File',
         category: :events
       }
     end
 
-    def read(url)
-      unless url.nil?
-        if url.to_s.downcase.end_with? 'sitemap.xml'
-          process_sitemap url
-        else
-          process_icalendar url
-        end
+    def read(source_url)
+      @verbose = false
+      sources = get_sources(source_url)
+      return if sources.nil?
+
+      sources.each do |url|
+        process_url(url)
       end
     end
 
     private
 
-    def process_sitemap(url)
-      # find urls for individual icalendar files
-      begin
-        sitemap = Nokogiri::XML.parse(open_url(url, raise: true))
-        locs = sitemap.xpath('/ns:urlset/ns:url/ns:loc', {
-                               'ns' => 'http://www.sitemaps.org/schemas/sitemap/0.9'
-                             })
-        locs.each do |loc|
-          process_icalendar(loc.text)
-        end
-      rescue Exception => e
-        @messages << "Extract from sitemap[#{url}] failed with: #{e.message}"
+    # Modifies the given URL to the ics or ical export.
+    # Loops into each Ical event to process it.
+    # Note: One .ics file can have multiple Ical events.
+    def process_url(url)
+      export_url = to_export(url)
+      events = Icalendar::Event.parse(open_url(export_url, raise: true).set_encoding('utf-8'))
+      events.each do |e|
+        process_calevent(e)
       end
+    rescue StandardError => e
+      @messages << "Process file url[#{export_url}] failed with: #{e.message}"
+    end
 
-      # finished
-      nil
+    # Builds the OpenStruct event and adds it in event.
+    def process_calevent(calevent)
+      event_to_add = OpenStruct.new.tap do |event|
+        assign_basic_info(event, calevent)
+        assign_time_info(event, calevent)
+        assign_location_info(event, calevent.location)
+      end
+      add_event(event_to_add)
+    rescue StandardError => e
+      @messages << "Process iCalendar failed with: #{e.message}"
     end
 
-    def process_icalendar(url)
-      # process individual ics file
-      query = '?ical=true'
+    # Assigns to event: url, title, description, keywords.
+    def assign_basic_info(event, calevent)
+      event.url = calevent.url.to_s
+      event.title = calevent.summary.to_s
+      event.description = process_description calevent.description
+      event.keywords = process_keywords(calevent.categories)
+    end
 
-      begin
-        # append query  (if required)
-        file_url = url
-        file_url << query unless url.to_s.downcase.ends_with? query
+    # Assigns to event: start, end, timezone.
+    def assign_time_info(event, calevent)
+      event.start = calevent.dtstart&.to_time unless calevent.dtstart.nil?
+      event.end = calevent.dtend&.to_time unless calevent.dtend.nil?
+      event.timezone = get_tzid(calevent.dtstart)
+    end
 
-        # process file
-        events = Icalendar::Event.parse(open_url(file_url, raise: true).set_encoding('utf-8'))
+    # Assigns to event: venue, online, city.
+    def assign_location_info(event, location)
+      return if location.blank? || !location.present?
 
-        # process each event
-        events.each do |e|
-          process_event(e)
-        end
-      rescue Exception => e
-        @messages << "Process file url[#{file_url}] failed with: #{e.message}"
-      end
+      event.venue = location.to_s
+      event.online = location.downcase.include?('online')
+      event.city, event.postcode, event.country = process_location(location)
+    end
 
-      # finished
-      nil
+    # Removes all `<br />` tags and converts HTML to MD.
+    def process_description(input)
+      return input if input.nil?
+
+      desc = input.to_s.gsub('', '<br />')
+      convert_description(desc)
     end
 
-    def process_event(calevent)
-      # puts "calevent: #{calevent.inspect}"
-      begin
-        # set fields
-        event = OpenStruct.new
-        event.url = calevent.url.to_s
-        event.title = calevent.summary.to_s
-        event.description = process_description calevent.description
-
-        # puts "\n\ncalevent.description = #{calevent.description}"
-        # puts "\n\n...        converted = #{event.description}"
-
-        event.end = calevent.dtend&.to_time
-        unless calevent.dtstart.nil?
-          dtstart = calevent.dtstart
-          event.start = dtstart&.to_time
-          tzid = dtstart.ical_params['tzid']
-          event.timezone = tzid.first.to_s if !tzid.nil? and tzid.size > 0
-        end
-
-        event.venue = calevent.location.to_s
-        if calevent.location.downcase.include?('online')
-          event.online = true
-          event.city = nil
-          event.postcode = nil
-          event.country = nil
-        else
-          location = convert_location(calevent.location)
-          event.city = location['suburb'] unless location['suburb'].nil?
-          event.country = location['country'] unless location['country'].nil?
-          event.postcode = location['postcode'] unless location['postcode'].nil?
-        end
-        event.keywords = []
-        unless calevent.categories.nil? or calevent.categories.first.nil?
-          cats = calevent.categories.first
-          if cats.is_a?(Icalendar::Values::Array)
-            cats.each do |item|
-              event.keywords << item.to_s.lstrip
-            end
-          else
-            event.keywords << cats.to_s.strip
-          end
-        end
-
-        # store event
-        @events << event
-      rescue Exception => e
-        @messages << "Process iCalendar failed with: #{e.message}"
-      end
+    # Extracts the timezone identifier (TZID) from an iCalendar event's dtstart field.
+    # Handles whether tzid shows up as an Array or a single string
+    def get_tzid(dtstart)
+      return nil unless dtstart.respond_to?(:ical_params)
 
-      # finished
-      nil
+      tzid = dtstart.ical_params['tzid']
+      return nil if tzid.nil?
+
+      tzid.is_a?(Array) ? tzid.first.to_s : tzid.to_s
     end
 
-    def process_description(input)
-      return input if input.nil?
+    # Returns an array of 3 location characteristics: suburb, postcode, country
+    # Everything is nil if location.blank or location is online
+    def process_location(location)
+      return [nil, nil, nil] if location.blank?
+
+      if location.to_s.downcase.include?('online')
+        [nil, nil, nil]
+      else
+        [
+          location['suburb'],
+          location['postcode'],
+          location['country']
+        ]
+      end
+    end
+
+    # Returns keywords from the `CATEGORIES` ICal field
+    def process_keywords(categories)
+      return [] if categories.blank?
 
-      convert_description(input.to_s.gsub(/\R/, '<br />'))
+      categories.flatten.compact.map { |cat| cat.to_s.strip }
     end
   end
 end

test/unit/ingestors/ical_ingestor_test.rb

@@ -26,7 +26,7 @@ class IcalIngestorTest < ActiveSupport::TestCase
 
     assert ingestor.events.empty?
     assert ingestor.materials.empty?
-    assert_includes ingestor.messages, 'Extract from sitemap[https://missing.org/sitemap.xml] failed with: 404 '
+    assert_includes ingestor.messages[0], 'Extract from sitemap[https://missing.org/sitemap.xml] failed with:'
   end
 
   test 'ingest valid sitemap' do
@@ -187,6 +187,58 @@ class IcalIngestorTest < ActiveSupport::TestCase
     end
   end
 
+  test 'process_calevent logs error when exception is raised' do
+    ingestor = Ingestors::IcalIngestor.new
+    calevent = Object.new # fake calevent
+
+    # Stub a method that will raise an error
+    def ingestor.assign_basic_info(*)
+      raise StandardError, 'test failure'
+    end
+
+    ingestor.send(:process_calevent, calevent)
+
+    assert_includes ingestor.messages.last, 'Process iCalendar failed with: test failure'
+  end
+
+  test 'to_export method' do
+    ingestor = Ingestors::IcalIngestor.new
+    indico_url_event = 'https://indico.cern.ch/event/1588342/'
+    indico_url_event_with_ics = 'https://indico.cern.ch/event/1588342/event.ics' # ! when '/event', event.ics is singular
+    indico_url_event_with_query = 'https://indico.cern.ch/event/1588342/?somerandom=urlparams&an=otherone'
+    indico_url_event_with_query_with_ics = 'https://indico.cern.ch/event/1588342/event.ics?somerandom=urlparams&an=otherone'
+    indico_url_category = 'https://indico.cern.ch/category/19377/'
+    indico_url_category_with_ics = 'https://indico.cern.ch/category/19377/events.ics' # ! when '/category', eventS.ics is plural
+    indico_url_category_with_query = 'https://indico.cern.ch/category/19377/?a=b&c=d'
+    indico_url_category_with_query_with_ics = 'https://indico.cern.ch/category/19377/events.ics?a=b&c=d'
+    url_with_ics = 'https://mywebsite.com/event/blabla/events.ics'
+    url_with_query_with_ics = 'https://mywebsite.com/event/blabla/events.ics?john=doe&isstub=born'
+    url_no_ical = 'https://mywebsite.com/event/blabla'
+    url_with_ical = 'https://mywebsite.com/event/blabla?ical=true'
+
+    # When indico link – event
+    assert_equal ingestor.send(:to_export, indico_url_event), indico_url_event_with_ics # adds ics
+    assert_equal ingestor.send(:to_export, indico_url_event_with_query), indico_url_event_with_query_with_ics # adds ics
+
+    # When indico link – category
+    assert_equal ingestor.send(:to_export, indico_url_category), indico_url_category_with_ics # adds ics
+    assert_equal ingestor.send(:to_export, indico_url_category_with_query), indico_url_category_with_query_with_ics # adds ics
+
+    # When non-indico link
+    assert_equal ingestor.send(:to_export, url_with_ics), url_with_ics # keeps same
+    assert_equal ingestor.send(:to_export, url_with_query_with_ics), url_with_query_with_ics # keeps same
+
+    # When indico link which already has the /events.ics
+    assert_equal ingestor.send(:to_export, indico_url_event_with_ics), indico_url_event_with_ics # keeps it as-is
+    assert_equal ingestor.send(:to_export, indico_url_event_with_query_with_ics), indico_url_event_with_query_with_ics # keeps it as-is
+
+    # When other url, adds the ical query param
+    assert_equal ingestor.send(:to_export, url_no_ical), url_with_ical
+
+    # When other url with ical query param, keep it as-is
+    assert_equal ingestor.send(:to_export, url_with_ical), url_with_ical
+  end
+
   private
 
   def check_event_exists(title, url)