From 975c1bc18149cb155097a9bbd01aa0bef65a67de Mon Sep 17 00:00:00 2001 From: kennethrioja <59597207+kennethrioja@users.noreply.github.com> Date: Fri, 13 Feb 2026 10:27:51 +0100 Subject: [PATCH 1/5] refactor(get_html_from_url): moved method from github ingestor to ingestor class --- lib/ingestors/github_ingestor.rb | 7 +------ lib/ingestors/ingestor.rb | 5 +++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/ingestors/github_ingestor.rb b/lib/ingestors/github_ingestor.rb index 79872f9a1..46dac41a6 100644 --- a/lib/ingestors/github_ingestor.rb +++ b/lib/ingestors/github_ingestor.rb @@ -104,7 +104,7 @@ def to_material(repo_data) # rubocop:disable Metrics/AbcSize github_io_homepage = github_io_homepage? repo_data['homepage'] url = github_io_homepage ? repo_data['homepage'] : repo_data['html_url'] redirected_url = get_redirected_url(url) - html = get_html(redirected_url) + html = get_html_from_url(redirected_url) material = OpenStruct.new material.title = repo_data['name'].titleize @@ -131,11 +131,6 @@ def github_io_homepage?(homepage) url.host&.downcase&.end_with?('.github.io') end - def get_html(url) - response = HTTParty.get(url, follow_redirects: true, headers: { 'User-Agent' => config[:user_agent] }) - Nokogiri::HTML(response.body) - end - # DEFINITION – Opens the GitHub homepage, fetches the 3 first >50 char

tags'text # and joins them with a 'Read more...' link at the end of the description # Some of the first

tags were not descriptive, thus skipping them diff --git a/lib/ingestors/ingestor.rb b/lib/ingestors/ingestor.rb index 241f5c59c..1e3de56de 100644 --- a/lib/ingestors/ingestor.rb +++ b/lib/ingestors/ingestor.rb @@ -84,6 +84,11 @@ def open_url(url, raise: false, token: nil) end end + def get_html_from_url(url) + response = HTTParty.get(url, follow_redirects: true, headers: { 'User-Agent' => config[:user_agent] }) + Nokogiri::HTML(response.body) + end + # Some URLs automatically redirects the user to another webpage # This method gets a URL and returns the last redirected URL (as shown by a 30X response or a `meta[http-equiv="Refresh"]` tag) def get_redirected_url(url, limit = 5) # rubocop:disable Metrics/AbcSize From 0e08094aa0b1167049a670aa1df7b952c7a0913a Mon Sep 17 00:00:00 2001 From: kennethrioja <59597207+kennethrioja@users.noreply.github.com> Date: Fri, 13 Feb 2026 10:29:26 +0100 Subject: [PATCH 2/5] feat(gray_scott_ingestor): added new event ingestor for heptraining --- .../heptraining/gray_scott_ingestor.rb | 75 +++++++++++++++++++ lib/ingestors/ingestor_factory.rb | 8 +- .../heptraining/grayscott/grayscott-event.ics | 23 ++++++ .../heptraining/grayscott/grayscott-page.html | 44 +++++++++++ .../grayscott/grayscott-redirect.html | 28 +++++++ .../heptraining/gray_scott_ingestor_test.rb | 40 ++++++++++ 6 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 lib/ingestors/heptraining/gray_scott_ingestor.rb create mode 100644 test/fixtures/files/ingestion/heptraining/grayscott/grayscott-event.ics create mode 100644 test/fixtures/files/ingestion/heptraining/grayscott/grayscott-page.html create mode 100644 test/fixtures/files/ingestion/heptraining/grayscott/grayscott-redirect.html create mode 100644 test/unit/ingestors/heptraining/gray_scott_ingestor_test.rb diff --git a/lib/ingestors/heptraining/gray_scott_ingestor.rb b/lib/ingestors/heptraining/gray_scott_ingestor.rb new file mode 100644 index 000000000..1faf05d67 --- /dev/null +++ b/lib/ingestors/heptraining/gray_scott_ingestor.rb @@ -0,0 +1,75 @@ +require 'icalendar' +require 'nokogiri' +require 'open-uri' +require 'tzinfo' + +module Ingestors + module Heptraining + class GrayScottIngestor < Ingestor + def self.config + { + key: 'gray_scott_event', + title: 'Gray Scott Events API', + category: :events + } + end + + def read(url) + @verbose = false + process_gray_scott(url) + end + + private + + def process_gray_scott(url) + events = Icalendar::Event.parse(open_url(url, raise: true).set_encoding('utf-8')) + raise 'Not found' if events.nil? || events.empty? + + events.each do |e| + process_calevent(e, url) + end + end + + def process_calevent(calevent, url) + # puts "calevent: #{calevent.inspect}" + gs_url = calevent.custom_properties.find { |key, _| key.include?('http') }&.last&.first&.strip&.gsub(%r{^[/\s]+|[/\s]+$}, '')&.prepend('https://') + html = get_html_from_url(get_redirected_url(gs_url)) + + event = OpenStruct.new + event.title = calevent.summary.to_s + event.url = gs_url + event.description = html.css('.paragraphStyle').text.strip || calevent.description.to_s + + event.end = calevent.dtend&.to_time + unless calevent.dtstart.nil? + dtstart = calevent.dtstart + event.start = dtstart&.to_time + tzid = dtstart.ical_params['tzid'] + event.timezone = tzid.first.to_s if !tzid.nil? && tzid.size.positive? + end + event.venue = clean_html(calevent.location.to_s) + event.organizer = html.css('h3:contains("Speakers") + ul li a').text.strip + + @events << event + end + + def get_redirected_url(url) + uri = URI.parse(url) + label = CGI.parse(uri.query)['label']&.first + + script_content = get_html_from_url(url).css('script').find { |s| s.content.include?('var dictReference') }&.content + dict_match = script_content&.match(/var\s+dictReference\s*=\s*({[^}]+})/) + return unless dict_match + + dict = JSON.parse(dict_match[1]) + matched_value = dict[label] + + "#{uri.scheme}://#{uri.host}#{uri.path.sub(%r{/[^/]+$}, '')}/#{matched_value}" + end + + def clean_html(html) + Nokogiri::HTML::DocumentFragment.parse(html).text.strip + end + end + end +end diff --git a/lib/ingestors/ingestor_factory.rb b/lib/ingestors/ingestor_factory.rb index 11d5a2151..4913e068d 100644 --- a/lib/ingestors/ingestor_factory.rb +++ b/lib/ingestors/ingestor_factory.rb @@ -12,7 +12,7 @@ def self.ingestors Ingestors::TessEventIngestor, Ingestors::ZenodoIngestor, Ingestors::GithubIngestor, - ] + taxila_ingestors + llm_ingestors + ] + taxila_ingestors + llm_ingestors + heptraining_ingestors end def self.taxila_ingestors @@ -49,6 +49,12 @@ def self.llm_ingestors ] end + def self.heptraining_ingestors + [ + Ingestors::Heptraining::GrayScottIngestor + ] + end + def self.ingestor_config @ingestor_config ||= ingestors.map do |i| [i.config[:key], i.config.merge(ingestor: i)] diff --git a/test/fixtures/files/ingestion/heptraining/grayscott/grayscott-event.ics b/test/fixtures/files/ingestion/heptraining/grayscott/grayscott-event.ics new file mode 100644 index 000000000..cb89b45c1 --- /dev/null +++ b/test/fixtures/files/ingestion/heptraining/grayscott/grayscott-event.ics @@ -0,0 +1,23 @@ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//PhoenixTex2Html//gray_scott_2026_webinars/ +BEGIN:VEVENT +CLASS:PUBLIC +DTSTAMP:20260212T103600 +UID:TH8WMR_PNR0012_20260212T103600 +DTSTART;TZID=Europe/Paris:20260226T100000 +DTEND;TZID=Europe/Paris:20260226T113000 +SUMMARY:Memory allocation, why and how to profile applications + +LOCATION:Registration : https://teratec.webex.com/blabla + +DESCRIPTION:Memory allocation, why and how to profile applications +\n + https://cta-lapp.pages.in2p3.fr/cours/gray_scott_revolutions/grayscott2026/redirect.html?label=sec_gray_scott_webinar_memory_allocation_memory_profiling\n +BEGIN:VALARM +TRIGGER:-PT10M +ACTION:DISPLAY +DESCRIPTION:Reminder +END:VALARM +END:VEVENT +END:VCALENDAR \ No newline at end of file diff --git a/test/fixtures/files/ingestion/heptraining/grayscott/grayscott-page.html b/test/fixtures/files/ingestion/heptraining/grayscott/grayscott-page.html new file mode 100644 index 000000000..a8396e194 --- /dev/null +++ b/test/fixtures/files/ingestion/heptraining/grayscott/grayscott-page.html @@ -0,0 +1,44 @@ + + + + + + Memory allocation, why and how to profile applications + + + + + + + + + + + + + + + + + + + + +

Date : 26/02/2026
+Location : Registration : https://teratec.webex.com/webappng/sites/teratec/webinar/webinarSeries/register/0465b64b919540de9910a5b84077b878 +
+Start at : 10:00
+Stop at : 11:30

Speakers

+ +

Description

+

+Sometimes memory has become a major problem in applications, with its bandwidth but also by the incresing size needed by more and more complex and dynamic applications. So, how to track these errors and point problematic patterns ? How to find where the memory is consumed when the application reaches the hardware limit ? After my PhD on memory management in HPC context (NUMA, parallel, etc) I had the opportunity to develop two profilers (malloc and numa) now open-sources for C/C++/Fortran and Rust. I will briefly present these tools with some examples and expected observations. +

+ + + + diff --git a/test/fixtures/files/ingestion/heptraining/grayscott/grayscott-redirect.html b/test/fixtures/files/ingestion/heptraining/grayscott/grayscott-redirect.html new file mode 100644 index 000000000..401d8bb1d --- /dev/null +++ b/test/fixtures/files/ingestion/heptraining/grayscott/grayscott-redirect.html @@ -0,0 +1,28 @@ + + + + + + Page redirection + + + + +
Dans 2 secondes vous allez être redirigé vers la page que vous avez demandée... normalement
+ + + diff --git a/test/unit/ingestors/heptraining/gray_scott_ingestor_test.rb b/test/unit/ingestors/heptraining/gray_scott_ingestor_test.rb new file mode 100644 index 000000000..1a2a26a6c --- /dev/null +++ b/test/unit/ingestors/heptraining/gray_scott_ingestor_test.rb @@ -0,0 +1,40 @@ +require 'test_helper' + +class GrayScottIngestorTest < ActiveSupport::TestCase + setup do + @ingestor = Ingestors::Heptraining::GrayScottIngestor.new + @user = users(:regular_user) + @content_provider = content_providers(:another_portal_provider) + + webmock('https://cta-lapp.pages.in2p3.fr/COURS/GRAY_SCOTT_REVOLUTIONS/GrayScott2026/invitation/gray_scott_2026_webinars.ics', 'heptraining/grayscott/grayscott-event.ics') + webmock('https://cta-lapp.pages.in2p3.fr/cours/gray_scott_revolutions/grayscott2026/redirect.html?label=sec_gray_scott_webinar_memory_allocation_memory_profiling', 'heptraining/grayscott/grayscott-redirect.html') + webmock('https://cta-lapp.pages.in2p3.fr/cours/gray_scott_revolutions/grayscott2026/1-1-5-1-449.html', 'heptraining/grayscott/grayscott-page.html') + end + + teardown do + reset_timezone + end + + test 'should read Gray Scott ics' do + @ingestor.read('https://cta-lapp.pages.in2p3.fr/COURS/GRAY_SCOTT_REVOLUTIONS/GrayScott2026/invitation/gray_scott_2026_webinars.ics') + @ingestor.write(@user, @content_provider) + + sample = @ingestor.events.detect { |e| e.title == 'Memory allocation, why and how to profile applications' } + assert sample.persisted? + + assert_equal sample.url, 'https://cta-lapp.pages.in2p3.fr/cours/gray_scott_revolutions/grayscott2026/redirect.html?label=sec_gray_scott_webinar_memory_allocation_memory_profiling' + assert_includes sample.description, 'Sometimes memory has become a major problem in applications' + assert_equal sample.end, '2026-02-26 10:30:00 +0000' + assert_equal sample.start, '2026-02-26 09:00:00 +0000' + assert_equal sample.timezone, 'Paris' + assert_includes sample.venue, 'teratec.webex.com' + assert_includes sample.organizer, 'Someone' + end + + private + + def webmock(url, filename) + file = Rails.root.join('test', 'fixtures', 'files', 'ingestion', filename) + WebMock.stub_request(:get, url).to_return(status: 200, headers: {}, body: file.read) + end +end From 747e500cf7413c1f1f3f91e2a67da4157ca6a6ef Mon Sep 17 00:00:00 2001 From: kennethrioja <59597207+kennethrioja@users.noreply.github.com> Date: Mon, 16 Feb 2026 12:09:13 +0100 Subject: [PATCH 3/5] fix(gray_scott_ingestor): handles multiple organizers --- lib/ingestors/heptraining/gray_scott_ingestor.rb | 2 +- .../files/ingestion/heptraining/grayscott/grayscott-page.html | 2 ++ test/unit/ingestors/heptraining/gray_scott_ingestor_test.rb | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/ingestors/heptraining/gray_scott_ingestor.rb b/lib/ingestors/heptraining/gray_scott_ingestor.rb index 1faf05d67..990837dbe 100644 --- a/lib/ingestors/heptraining/gray_scott_ingestor.rb +++ b/lib/ingestors/heptraining/gray_scott_ingestor.rb @@ -48,7 +48,7 @@ def process_calevent(calevent, url) event.timezone = tzid.first.to_s if !tzid.nil? && tzid.size.positive? end event.venue = clean_html(calevent.location.to_s) - event.organizer = html.css('h3:contains("Speakers") + ul li a').text.strip + event.organizer = html.css('h3:contains("Speakers") + ul li a')&.map(&:text)&.map(&:strip)&.join(', ') # coma separated if multiple speakers @events << event end diff --git a/test/fixtures/files/ingestion/heptraining/grayscott/grayscott-page.html b/test/fixtures/files/ingestion/heptraining/grayscott/grayscott-page.html index a8396e194..c7d909413 100644 --- a/test/fixtures/files/ingestion/heptraining/grayscott/grayscott-page.html +++ b/test/fixtures/files/ingestion/heptraining/grayscott/grayscott-page.html @@ -32,6 +32,8 @@ Stop at : 11:30

Speakers

Description

diff --git a/test/unit/ingestors/heptraining/gray_scott_ingestor_test.rb b/test/unit/ingestors/heptraining/gray_scott_ingestor_test.rb index 1a2a26a6c..534d1fe91 100644 --- a/test/unit/ingestors/heptraining/gray_scott_ingestor_test.rb +++ b/test/unit/ingestors/heptraining/gray_scott_ingestor_test.rb @@ -28,7 +28,7 @@ class GrayScottIngestorTest < ActiveSupport::TestCase assert_equal sample.start, '2026-02-26 09:00:00 +0000' assert_equal sample.timezone, 'Paris' assert_includes sample.venue, 'teratec.webex.com' - assert_includes sample.organizer, 'Someone' + assert_equal sample.organizer, 'Someone, SomeoneElse' end private From 6cb0ae32a0751623cf0c94779add98d672eba384 Mon Sep 17 00:00:00 2001 From: kennethrioja <59597207+kennethrioja@users.noreply.github.com> Date: Fri, 20 Feb 2026 14:28:48 +0100 Subject: [PATCH 4/5] fix(gray_scott_ingestor): start and end to utc --- lib/ingestors/heptraining/gray_scott_ingestor.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ingestors/heptraining/gray_scott_ingestor.rb b/lib/ingestors/heptraining/gray_scott_ingestor.rb index 990837dbe..6477b0aa0 100644 --- a/lib/ingestors/heptraining/gray_scott_ingestor.rb +++ b/lib/ingestors/heptraining/gray_scott_ingestor.rb @@ -40,10 +40,10 @@ def process_calevent(calevent, url) event.url = gs_url event.description = html.css('.paragraphStyle').text.strip || calevent.description.to_s - event.end = calevent.dtend&.to_time + event.end = calevent.dtend&.to_time&.utc unless calevent.dtstart.nil? dtstart = calevent.dtstart - event.start = dtstart&.to_time + event.start = dtstart&.to_time&.utc tzid = dtstart.ical_params['tzid'] event.timezone = tzid.first.to_s if !tzid.nil? && tzid.size.positive? end From ebd45cbf3f0b401c6489faa5f6940b75816d9eff Mon Sep 17 00:00:00 2001 From: kennethrioja <59597207+kennethrioja@users.noreply.github.com> Date: Fri, 6 Mar 2026 16:33:57 +0100 Subject: [PATCH 5/5] review(gray-scott-ingestor): from copilot --- lib/ingestors/heptraining/gray_scott_ingestor.rb | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/ingestors/heptraining/gray_scott_ingestor.rb b/lib/ingestors/heptraining/gray_scott_ingestor.rb index 6477b0aa0..386a00866 100644 --- a/lib/ingestors/heptraining/gray_scott_ingestor.rb +++ b/lib/ingestors/heptraining/gray_scott_ingestor.rb @@ -10,7 +10,8 @@ def self.config { key: 'gray_scott_event', title: 'Gray Scott Events API', - category: :events + category: :events, + user_agent: 'TeSS Gray Scott ingestor' } end @@ -33,12 +34,13 @@ def process_gray_scott(url) def process_calevent(calevent, url) # puts "calevent: #{calevent.inspect}" gs_url = calevent.custom_properties.find { |key, _| key.include?('http') }&.last&.first&.strip&.gsub(%r{^[/\s]+|[/\s]+$}, '')&.prepend('https://') - html = get_html_from_url(get_redirected_url(gs_url)) + html = get_html_from_url(get_gray_scott_redirection(gs_url)) event = OpenStruct.new event.title = calevent.summary.to_s event.url = gs_url - event.description = html.css('.paragraphStyle').text.strip || calevent.description.to_s + html_description = html.css('.paragraphStyle').text.to_s.strip + event.description = html_description.empty? ? calevent.description.to_s : html_description event.end = calevent.dtend&.to_time&.utc unless calevent.dtstart.nil? @@ -53,7 +55,7 @@ def process_calevent(calevent, url) @events << event end - def get_redirected_url(url) + def get_gray_scott_redirection(url) uri = URI.parse(url) label = CGI.parse(uri.query)['label']&.first @@ -63,6 +65,7 @@ def get_redirected_url(url) dict = JSON.parse(dict_match[1]) matched_value = dict[label] + return url unless matched_value "#{uri.scheme}://#{uri.host}#{uri.path.sub(%r{/[^/]+$}, '')}/#{matched_value}" end