|
1 | | -# frozen_string_literal: true |
2 | | - |
3 | 1 | require 'icalendar' |
4 | 2 | require 'nokogiri' |
5 | 3 | require 'open-uri' |
6 | 4 | require 'tzinfo' |
7 | 5 |
|
8 | 6 | module Ingestors |
9 | | - # Reads from direct ical / .ics / Indico (event or category) URLs, .xml sitemaps, and .txt sitemaps. |
10 | 7 | class IcalIngestor < Ingestor |
11 | | - include Ingestors::Concerns::SitemapHelpers |
12 | | - |
13 | 8 | def self.config |
14 | 9 | { |
15 | 10 | key: 'ical', |
16 | | - title: 'iCalendar / Indico / .ics File', |
| 11 | + title: 'iCalendar', |
17 | 12 | category: :events |
18 | 13 | } |
19 | 14 | end |
20 | 15 |
|
21 | | - def read(source_url) |
22 | | - @token = Rails.application.config.secrets.indico_api_token |
23 | | - @verbose = false |
24 | | - sources = get_sources(source_url) |
25 | | - return if sources.nil? |
26 | | - |
27 | | - sources.each do |url| |
28 | | - process_url(url) |
| 16 | + def read(url) |
| 17 | + unless url.nil? |
| 18 | + if url.to_s.downcase.end_with? 'sitemap.xml' |
| 19 | + process_sitemap url |
| 20 | + else |
| 21 | + process_icalendar url |
| 22 | + end |
29 | 23 | end |
30 | 24 | end |
31 | 25 |
|
32 | 26 | private |
33 | 27 |
|
34 | | - # Modifies the given URL to the ics or ical export. |
35 | | - # Loops into each Ical event to process it. |
36 | | - # Note: One .ics file can have multiple Ical events. |
37 | | - def process_url(url) |
38 | | - export_url = to_export(url) |
39 | | - content = open_url(export_url, token: @token, raise: true).set_encoding('utf-8') |
40 | | - events = Icalendar::Event.parse(content) |
41 | | - raise 'Not found' if events.nil? || events.empty? |
42 | | - |
43 | | - events.each do |e| |
44 | | - process_calevent(e) |
45 | | - end |
46 | | - rescue StandardError => e |
47 | | - @messages << "Process file url[#{export_url}] failed with: #{e.message}" |
48 | | - end |
49 | | - |
50 | | - # 1. If the path already ends with '/events.ics', return as-is. |
51 | | - # 2. If the host includes 'indico', ensures the path ends with '/events.ics'. |
52 | | - # 3. Otherwise, append '?ical=true' query param if not already present. |
53 | | - # |
54 | | - # This method never mutates the original URL string. |
55 | | - # Returns the updated URL string or nil if input is blank. |
56 | | - def to_export(url) |
57 | | - return nil if url.blank? |
58 | | - |
59 | | - uri = URI.parse(url) |
60 | | - path = uri.path.to_s |
61 | | - |
62 | | - if path.match?(%r{/(event|events)\.ics\z}) |
63 | | - uri.to_s |
64 | | - elsif uri.host&.include?('indico') |
65 | | - ensure_events_ics_path(uri) |
66 | | - else |
67 | | - ensure_ical_query(uri) |
68 | | - end |
69 | | - end |
70 | | - |
71 | | - # Ensures the Indico URL ends with '/events.ics' |
72 | | - def ensure_events_ics_path(uri) |
73 | | - paths = uri.path.split('/') |
74 | | - uri.path = "#{paths[0..2].join('/')}/" |
75 | | - if paths[1] == 'event' |
76 | | - uri.path = File.join(uri.path, 'event.ics') |
77 | | - elsif paths[1] == 'category' |
78 | | - uri.path = File.join(uri.path, 'events.ics') |
79 | | - end |
80 | | - uri.to_s |
81 | | - end |
82 | | - |
83 | | - # Ensures the URL has '?ical=true' in its query params |
84 | | - def ensure_ical_query(uri) |
85 | | - query = URI.decode_www_form(uri.query.to_s).to_h |
86 | | - query['ical'] = 'true' unless query['ical'] == 'true' |
87 | | - uri.query = URI.encode_www_form(query) |
88 | | - uri.to_s |
89 | | - end |
90 | | - |
91 | | - # Builds the OpenStruct event and adds it in event. |
92 | | - def process_calevent(calevent) |
93 | | - event_to_add = OpenStruct.new.tap do |event| |
94 | | - assign_basic_info(event, calevent) |
95 | | - assign_time_info(event, calevent) |
96 | | - assign_location_info(event, calevent.location) |
| 28 | + def process_sitemap(url) |
| 29 | + # find urls for individual icalendar files |
| 30 | + begin |
| 31 | + sitemap = Nokogiri::XML.parse(open_url(url, raise: true)) |
| 32 | + locs = sitemap.xpath('/ns:urlset/ns:url/ns:loc', { |
| 33 | + 'ns' => 'http://www.sitemaps.org/schemas/sitemap/0.9' |
| 34 | + }) |
| 35 | + locs.each do |loc| |
| 36 | + process_icalendar(loc.text) |
| 37 | + end |
| 38 | + rescue Exception => e |
| 39 | + @messages << "Extract from sitemap[#{url}] failed with: #{e.message}" |
97 | 40 | end |
98 | | - add_event(event_to_add) |
99 | | - rescue StandardError => e |
100 | | - @messages << "Process iCalendar failed with: #{e.message}" |
101 | | - end |
102 | 41 |
|
103 | | - # Assigns to event: url, title, description, keywords. |
104 | | - def assign_basic_info(event, calevent) |
105 | | - event.url = calevent.url.to_s |
106 | | - event.title = calevent.summary.to_s |
107 | | - event.description = calevent.description.to_s |
108 | | - event.keywords = process_keywords(calevent.categories) |
109 | | - event.contact = calevent.contact.join(', ') |
| 42 | + # finished |
| 43 | + nil |
110 | 44 | end |
111 | 45 |
|
112 | | - # Assigns to event: start, end, timezone. |
113 | | - def assign_time_info(event, calevent) |
114 | | - event.start = calevent.dtstart&.to_time unless calevent.dtstart.nil? |
115 | | - event.end = calevent.dtend&.to_time unless calevent.dtend.nil? |
116 | | - event.timezone = get_tzid(calevent.dtstart) |
117 | | - end |
118 | | - |
119 | | - # Assigns to event: venue, online, city. |
120 | | - def assign_location_info(event, location) |
121 | | - return if location.blank? |
| 46 | + def process_icalendar(url) |
| 47 | + # process individual ics file |
| 48 | + query = '?ical=true' |
122 | 49 |
|
123 | | - event.venue = location.to_s |
124 | | - event.online = location.downcase.include?('online') |
125 | | - event.city, event.postcode, event.country = process_location(location) |
126 | | - end |
| 50 | + begin |
| 51 | + # append query (if required) |
| 52 | + file_url = url |
| 53 | + file_url << query unless url.to_s.downcase.ends_with? query |
127 | 54 |
|
128 | | - # Extracts the timezone identifier (TZID) from an iCalendar event's dtstart field. |
129 | | - # Handles whether tzid shows up as an Array or a single string |
130 | | - def get_tzid(dtstart) |
131 | | - return nil unless dtstart.respond_to?(:ical_params) |
| 55 | + # process file |
| 56 | + events = Icalendar::Event.parse(open_url(file_url, raise: true).set_encoding('utf-8')) |
132 | 57 |
|
133 | | - tzid = dtstart.ical_params['tzid'] |
134 | | - return nil if tzid.nil? |
| 58 | + # process each event |
| 59 | + events.each do |e| |
| 60 | + process_event(e) |
| 61 | + end |
| 62 | + rescue Exception => e |
| 63 | + @messages << "Process file url[#{file_url}] failed with: #{e.message}" |
| 64 | + end |
135 | 65 |
|
136 | | - tzid.is_a?(Array) ? tzid.first.to_s : tzid.to_s |
| 66 | + # finished |
| 67 | + nil |
137 | 68 | end |
138 | 69 |
|
139 | | - # Returns an array of 3 location characteristics: suburb, postcode, country |
140 | | - # Everything is nil if location.blank or location is online |
141 | | - def process_location(location) |
142 | | - return [location['suburb'], location['postcode'], location['country']] if location.is_a?(Array) |
| 70 | + def process_event(calevent) |
| 71 | + # puts "calevent: #{calevent.inspect}" |
| 72 | + begin |
| 73 | + # set fields |
| 74 | + event = OpenStruct.new |
| 75 | + event.url = calevent.url.to_s |
| 76 | + event.title = calevent.summary.to_s |
| 77 | + event.description = process_description calevent.description |
| 78 | + |
| 79 | + # puts "\n\ncalevent.description = #{calevent.description}" |
| 80 | + # puts "\n\n... converted = #{event.description}" |
| 81 | + |
| 82 | + event.end = calevent.dtend&.to_time |
| 83 | + unless calevent.dtstart.nil? |
| 84 | + dtstart = calevent.dtstart |
| 85 | + event.start = dtstart&.to_time |
| 86 | + tzid = dtstart.ical_params['tzid'] |
| 87 | + event.timezone = tzid.first.to_s if !tzid.nil? and tzid.size > 0 |
| 88 | + end |
| 89 | + |
| 90 | + event.venue = calevent.location.to_s |
| 91 | + if calevent.location.downcase.include?('online') |
| 92 | + event.online = true |
| 93 | + event.city = nil |
| 94 | + event.postcode = nil |
| 95 | + event.country = nil |
| 96 | + else |
| 97 | + location = convert_location(calevent.location) |
| 98 | + event.city = location['suburb'] unless location['suburb'].nil? |
| 99 | + event.country = location['country'] unless location['country'].nil? |
| 100 | + event.postcode = location['postcode'] unless location['postcode'].nil? |
| 101 | + end |
| 102 | + event.keywords = [] |
| 103 | + unless calevent.categories.nil? or calevent.categories.first.nil? |
| 104 | + cats = calevent.categories.first |
| 105 | + if cats.is_a?(Icalendar::Values::Array) |
| 106 | + cats.each do |item| |
| 107 | + event.keywords << item.to_s.lstrip |
| 108 | + end |
| 109 | + else |
| 110 | + event.keywords << cats.to_s.strip |
| 111 | + end |
| 112 | + end |
| 113 | + |
| 114 | + # store event |
| 115 | + @events << event |
| 116 | + rescue Exception => e |
| 117 | + @messages << "Process iCalendar failed with: #{e.message}" |
| 118 | + end |
143 | 119 |
|
144 | | - [nil, nil, nil] |
| 120 | + # finished |
| 121 | + nil |
145 | 122 | end |
146 | 123 |
|
147 | | - # Returns keywords from the `CATEGORIES` ICal field |
148 | | - def process_keywords(categories) |
149 | | - return [] if categories.blank? |
| 124 | + def process_description(input) |
| 125 | + return input if input.nil? |
150 | 126 |
|
151 | | - categories.flatten.compact.map { |cat| cat.to_s.strip } |
| 127 | + convert_description(input.to_s.gsub(/\R/, '<br />')) |
152 | 128 | end |
153 | 129 | end |
154 | 130 | end |
0 commit comments