tags'text # and joins them with a 'Read more...' link at the end of the description # Some of the first
tags were not descriptive, thus skipping them diff --git a/lib/ingestors/heptraining/gray_scott_ingestor.rb b/lib/ingestors/heptraining/gray_scott_ingestor.rb new file mode 100644 index 000000000..386a00866 --- /dev/null +++ b/lib/ingestors/heptraining/gray_scott_ingestor.rb @@ -0,0 +1,78 @@ +require 'icalendar' +require 'nokogiri' +require 'open-uri' +require 'tzinfo' + +module Ingestors + module Heptraining + class GrayScottIngestor < Ingestor + def self.config + { + key: 'gray_scott_event', + title: 'Gray Scott Events API', + category: :events, + user_agent: 'TeSS Gray Scott ingestor' + } + end + + def read(url) + @verbose = false + process_gray_scott(url) + end + + private + + def process_gray_scott(url) + events = Icalendar::Event.parse(open_url(url, raise: true).set_encoding('utf-8')) + raise 'Not found' if events.nil? || events.empty? + + events.each do |e| + process_calevent(e, url) + end + end + + def process_calevent(calevent, url) + # puts "calevent: #{calevent.inspect}" + gs_url = calevent.custom_properties.find { |key, _| key.include?('http') }&.last&.first&.strip&.gsub(%r{^[/\s]+|[/\s]+$}, '')&.prepend('https://') + html = get_html_from_url(get_gray_scott_redirection(gs_url)) + + event = OpenStruct.new + event.title = calevent.summary.to_s + event.url = gs_url + html_description = html.css('.paragraphStyle').text.to_s.strip + event.description = html_description.empty? ? calevent.description.to_s : html_description + + event.end = calevent.dtend&.to_time&.utc + unless calevent.dtstart.nil? + dtstart = calevent.dtstart + event.start = dtstart&.to_time&.utc + tzid = dtstart.ical_params['tzid'] + event.timezone = tzid.first.to_s if !tzid.nil? && tzid.size.positive? + end + event.venue = clean_html(calevent.location.to_s) + event.organizer = html.css('h3:contains("Speakers") + ul li a')&.map(&:text)&.map(&:strip)&.join(', ') # coma separated if multiple speakers + + @events << event + end + + def get_gray_scott_redirection(url) + uri = URI.parse(url) + label = CGI.parse(uri.query)['label']&.first + + script_content = get_html_from_url(url).css('script').find { |s| s.content.include?('var dictReference') }&.content + dict_match = script_content&.match(/var\s+dictReference\s*=\s*({[^}]+})/) + return unless dict_match + + dict = JSON.parse(dict_match[1]) + matched_value = dict[label] + return url unless matched_value + + "#{uri.scheme}://#{uri.host}#{uri.path.sub(%r{/[^/]+$}, '')}/#{matched_value}" + end + + def clean_html(html) + Nokogiri::HTML::DocumentFragment.parse(html).text.strip + end + end + end +end diff --git a/lib/ingestors/ingestor.rb b/lib/ingestors/ingestor.rb index 241f5c59c..1e3de56de 100644 --- a/lib/ingestors/ingestor.rb +++ b/lib/ingestors/ingestor.rb @@ -84,6 +84,11 @@ def open_url(url, raise: false, token: nil) end end + def get_html_from_url(url) + response = HTTParty.get(url, follow_redirects: true, headers: { 'User-Agent' => config[:user_agent] }) + Nokogiri::HTML(response.body) + end + # Some URLs automatically redirects the user to another webpage # This method gets a URL and returns the last redirected URL (as shown by a 30X response or a `meta[http-equiv="Refresh"]` tag) def get_redirected_url(url, limit = 5) # rubocop:disable Metrics/AbcSize diff --git a/lib/ingestors/ingestor_factory.rb b/lib/ingestors/ingestor_factory.rb index 11d5a2151..4913e068d 100644 --- a/lib/ingestors/ingestor_factory.rb +++ b/lib/ingestors/ingestor_factory.rb @@ -12,7 +12,7 @@ def self.ingestors Ingestors::TessEventIngestor, Ingestors::ZenodoIngestor, Ingestors::GithubIngestor, - ] + taxila_ingestors + llm_ingestors + ] + taxila_ingestors + llm_ingestors + heptraining_ingestors end def self.taxila_ingestors @@ -49,6 +49,12 @@ def self.llm_ingestors ] end + def self.heptraining_ingestors + [ + Ingestors::Heptraining::GrayScottIngestor + ] + end + def self.ingestor_config @ingestor_config ||= ingestors.map do |i| [i.config[:key], i.config.merge(ingestor: i)] diff --git a/test/fixtures/files/ingestion/heptraining/grayscott/grayscott-event.ics b/test/fixtures/files/ingestion/heptraining/grayscott/grayscott-event.ics new file mode 100644 index 000000000..cb89b45c1 --- /dev/null +++ b/test/fixtures/files/ingestion/heptraining/grayscott/grayscott-event.ics @@ -0,0 +1,23 @@ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//PhoenixTex2Html//gray_scott_2026_webinars/ +BEGIN:VEVENT +CLASS:PUBLIC +DTSTAMP:20260212T103600 +UID:TH8WMR_PNR0012_20260212T103600 +DTSTART;TZID=Europe/Paris:20260226T100000 +DTEND;TZID=Europe/Paris:20260226T113000 +SUMMARY:Memory allocation, why and how to profile applications + +LOCATION:Registration : https://teratec.webex.com/blabla + +DESCRIPTION:Memory allocation, why and how to profile applications +\n + https://cta-lapp.pages.in2p3.fr/cours/gray_scott_revolutions/grayscott2026/redirect.html?label=sec_gray_scott_webinar_memory_allocation_memory_profiling\n +BEGIN:VALARM +TRIGGER:-PT10M +ACTION:DISPLAY +DESCRIPTION:Reminder +END:VALARM +END:VEVENT +END:VCALENDAR \ No newline at end of file diff --git a/test/fixtures/files/ingestion/heptraining/grayscott/grayscott-page.html b/test/fixtures/files/ingestion/heptraining/grayscott/grayscott-page.html new file mode 100644 index 000000000..c7d909413 --- /dev/null +++ b/test/fixtures/files/ingestion/heptraining/grayscott/grayscott-page.html @@ -0,0 +1,46 @@ + + + +
+ ++Sometimes memory has become a major problem in applications, with its bandwidth but also by the incresing size needed by more and more complex and dynamic applications. So, how to track these errors and point problematic patterns ? How to find where the memory is consumed when the application reaches the hardware limit ? After my PhD on memory management in HPC context (NUMA, parallel, etc) I had the opportunity to develop two profilers (malloc and numa) now open-sources for C/C++/Fortran and Rust. I will briefly present these tools with some examples and expected observations. +
+ + + + diff --git a/test/fixtures/files/ingestion/heptraining/grayscott/grayscott-redirect.html b/test/fixtures/files/ingestion/heptraining/grayscott/grayscott-redirect.html new file mode 100644 index 000000000..401d8bb1d --- /dev/null +++ b/test/fixtures/files/ingestion/heptraining/grayscott/grayscott-redirect.html @@ -0,0 +1,28 @@ + + + + + +