Skip to content

Commit 0e08094

Browse files
kennethriojakennethrioja
committed
feat(gray_scott_ingestor): added new event ingestor for heptraining
1 parent 975c1bc commit 0e08094

6 files changed

Lines changed: 217 additions & 1 deletion

File tree

‎lib/ingestors/heptraining/gray_scott_ingestor.rb‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
require 'icalendar'
2+
require 'nokogiri'
3+
require 'open-uri'
4+
require 'tzinfo'
5+
6+
module Ingestors
7+
module Heptraining
8+
class GrayScottIngestor < Ingestor
9+
def self.config
10+
{
11+
key: 'gray_scott_event',
12+
title: 'Gray Scott Events API',
13+
category: :events
14+
}
15+
end
16+
17+
def read(url)
18+
@verbose = false
19+
process_gray_scott(url)
20+
end
21+
22+
private
23+
24+
def process_gray_scott(url)
25+
events = Icalendar::Event.parse(open_url(url, raise: true).set_encoding('utf-8'))
26+
raise 'Not found' if events.nil? || events.empty?
27+
28+
events.each do |e|
29+
process_calevent(e, url)
30+
end
31+
end
32+
33+
def process_calevent(calevent, url)
34+
# puts "calevent: #{calevent.inspect}"
35+
gs_url = calevent.custom_properties.find { |key, _| key.include?('http') }&.last&.first&.strip&.gsub(%r{^[/\s]+|[/\s]+$}, '')&.prepend('https://')
36+
html = get_html_from_url(get_redirected_url(gs_url))
37+
38+
event = OpenStruct.new
39+
event.title = calevent.summary.to_s
40+
event.url = gs_url
41+
event.description = html.css('.paragraphStyle').text.strip || calevent.description.to_s
42+
43+
event.end = calevent.dtend&.to_time
44+
unless calevent.dtstart.nil?
45+
dtstart = calevent.dtstart
46+
event.start = dtstart&.to_time
47+
tzid = dtstart.ical_params['tzid']
48+
event.timezone = tzid.first.to_s if !tzid.nil? && tzid.size.positive?
49+
end
50+
event.venue = clean_html(calevent.location.to_s)
51+
event.organizer = html.css('h3:contains("Speakers") + ul li a').text.strip
52+
53+
@events << event
54+
end
55+
56+
def get_redirected_url(url)
57+
uri = URI.parse(url)
58+
label = CGI.parse(uri.query)['label']&.first
59+
60+
script_content = get_html_from_url(url).css('script').find { |s| s.content.include?('var dictReference') }&.content
61+
dict_match = script_content&.match(/var\s+dictReference\s*=\s*({[^}]+})/)
62+
return unless dict_match
63+
64+
dict = JSON.parse(dict_match[1])
65+
matched_value = dict[label]
66+
67+
"#{uri.scheme}://#{uri.host}#{uri.path.sub(%r{/[^/]+$}, '')}/#{matched_value}"
68+
end
69+
70+
def clean_html(html)
71+
Nokogiri::HTML::DocumentFragment.parse(html).text.strip
72+
end
73+
end
74+
end
75+
end

‎lib/ingestors/ingestor_factory.rb‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ def self.ingestors
1212
Ingestors::TessEventIngestor,
1313
Ingestors::ZenodoIngestor,
1414
Ingestors::GithubIngestor,
15-
] + taxila_ingestors + llm_ingestors
15+
] + taxila_ingestors + llm_ingestors + heptraining_ingestors
1616
end
1717

1818
def self.taxila_ingestors
@@ -49,6 +49,12 @@ def self.llm_ingestors
4949
]
5050
end
5151

52+
def self.heptraining_ingestors
53+
[
54+
Ingestors::Heptraining::GrayScottIngestor
55+
]
56+
end
57+
5258
def self.ingestor_config
5359
@ingestor_config ||= ingestors.map do |i|
5460
[i.config[:key], i.config.merge(ingestor: i)]

‎test/fixtures/files/ingestion/heptraining/grayscott/grayscott-event.ics‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
BEGIN:VCALENDAR
2+
VERSION:2.0
3+
PRODID:-//PhoenixTex2Html//gray_scott_2026_webinars/
4+
BEGIN:VEVENT
5+
CLASS:PUBLIC
6+
DTSTAMP:20260212T103600
7+
UID:TH8WMR_PNR0012_20260212T103600
8+
DTSTART;TZID=Europe/Paris:20260226T100000
9+
DTEND;TZID=Europe/Paris:20260226T113000
10+
SUMMARY:Memory allocation, why and how to profile applications
11+
12+
LOCATION:Registration : <a id="0" href="https://teratec.webex.com/blabla">https://teratec.webex.com/blabla</a>
13+
14+
DESCRIPTION:Memory allocation, why and how to profile applications
15+
\n
16+
https://cta-lapp.pages.in2p3.fr/cours/gray_scott_revolutions/grayscott2026/redirect.html?label=sec_gray_scott_webinar_memory_allocation_memory_profiling\n
17+
BEGIN:VALARM
18+
TRIGGER:-PT10M
19+
ACTION:DISPLAY
20+
DESCRIPTION:Reminder
21+
END:VALARM
22+
END:VEVENT
23+
END:VCALENDAR

‎test/fixtures/files/ingestion/heptraining/grayscott/grayscott-page.html‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
2+
<!DOCTYPE html>
3+
<html class="js sidebar-visible navy" lang="fr">
4+
<head>
5+
<meta charset="UTF-8">
6+
<title>Memory allocation, why and how to profile applications
7+
</title>
8+
<meta content="text/html; charset=UTF-8" http-equiv="Content-Type">
9+
<meta name="description" content="Memory allocation, why and how to profile applications
10+
">
11+
<meta name="viewport" content="width=device-width, initial-scale=1">
12+
<meta name="theme-color" content="rgba(0, 0, 0, 0)">
13+
<link rel="stylesheet" href="variables.css">
14+
<link rel="stylesheet" href="dark_style.css" />
15+
<link rel="stylesheet" href="general.css">
16+
<link rel="stylesheet" href="chrome.css">
17+
<link rel="stylesheet" href="highlight.css" disabled="">
18+
<link rel="stylesheet" href="tomorrow-night.css">
19+
<link rel="stylesheet" href="ayu-highlight.css" disabled="">
20+
<!-- Fonts -->
21+
<link rel="stylesheet" href="font-awesome.css">
22+
<link rel="stylesheet" href="fonts.css">
23+
<!-- <script src="" async></script> -->
24+
<!-- <script src=""></script> -->
25+
</head>
26+
<body>
27+
28+
<a id="450" href="invitation/gray_scott_webinar_memory_allocation_memory_profiling.ics"><div class="rendezvousStyle"></div></a><b>Date</b> : 26/02/2026<br />
29+
<b>Location</b> : Registration : <a id="458" href="https://teratec.webex.com/webappng/sites/teratec/webinar/webinarSeries/register/0465b64b919540de9910a5b84077b878">https://teratec.webex.com/webappng/sites/teratec/webinar/webinarSeries/register/0465b64b919540de9910a5b84077b878</a>
30+
<br />
31+
<b>Start at</b> : 10:00<br />
32+
<b>Stop at</b> : 11:30 <h3 id="466">Speakers</h3>
33+
<ul>
34+
<li><a href="2-3-5-4513.html">Someone
35+
</a></li>
36+
</ul>
37+
<h3 id="471">Description</h3>
38+
<p id="472" class="paragraphStyle">
39+
Sometimes memory has become a major problem in applications, with its bandwidth but also by the incresing size needed by more and more complex and dynamic applications. So, how to track these errors and point problematic patterns ? How to find where the memory is consumed when the application reaches the hardware limit ? After my PhD on memory management in HPC context (NUMA, parallel, etc) I had the opportunity to develop two profilers (malloc and numa) now open-sources for C/C++/Fortran and Rust. I will briefly present these tools with some examples and expected observations.
40+
</p>
41+
42+
</body>
43+
</html>
44+

‎test/fixtures/files/ingestion/heptraining/grayscott/grayscott-redirect.html‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
2+
<!DOCTYPE html>
3+
<html lang="fr">
4+
<head>
5+
<meta charset="utf-8" />
6+
<title>Page redirection</title>
7+
<link rel="stylesheet" href="dark_style.css" />
8+
<script type="text/javascript">
9+
function redirectionWithLabelReference(){
10+
var parameters = location.search.substring(1).split("?");
11+
var tmp = parameters[0].split("=");
12+
referenceName = unescape(tmp[1]);
13+
var dictReference = {
14+
"sec_gray_scott_webinar_memory_allocation_memory_profiling": "1-1-5-1-449.html"
15+
};
16+
if(referenceName in dictReference){
17+
document.location.href=dictReference[referenceName];
18+
}else{
19+
document.location.href="index.html";
20+
}
21+
}
22+
</script>
23+
</head>
24+
<body onLoad="setTimeout('redirectionWithLabelReference()', 1000)">
25+
<div>Dans 2 secondes vous allez être redirigé vers la page que vous avez demandée... normalement</div>
26+
</body>
27+
</html>
28+

‎test/unit/ingestors/heptraining/gray_scott_ingestor_test.rb‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
require 'test_helper'
2+
3+
class GrayScottIngestorTest < ActiveSupport::TestCase
4+
setup do
5+
@ingestor = Ingestors::Heptraining::GrayScottIngestor.new
6+
@user = users(:regular_user)
7+
@content_provider = content_providers(:another_portal_provider)
8+
9+
webmock('https://cta-lapp.pages.in2p3.fr/COURS/GRAY_SCOTT_REVOLUTIONS/GrayScott2026/invitation/gray_scott_2026_webinars.ics', 'heptraining/grayscott/grayscott-event.ics')
10+
webmock('https://cta-lapp.pages.in2p3.fr/cours/gray_scott_revolutions/grayscott2026/redirect.html?label=sec_gray_scott_webinar_memory_allocation_memory_profiling', 'heptraining/grayscott/grayscott-redirect.html')
11+
webmock('https://cta-lapp.pages.in2p3.fr/cours/gray_scott_revolutions/grayscott2026/1-1-5-1-449.html', 'heptraining/grayscott/grayscott-page.html')
12+
end
13+
14+
teardown do
15+
reset_timezone
16+
end
17+
18+
test 'should read Gray Scott ics' do
19+
@ingestor.read('https://cta-lapp.pages.in2p3.fr/COURS/GRAY_SCOTT_REVOLUTIONS/GrayScott2026/invitation/gray_scott_2026_webinars.ics')
20+
@ingestor.write(@user, @content_provider)
21+
22+
sample = @ingestor.events.detect { |e| e.title == 'Memory allocation, why and how to profile applications' }
23+
assert sample.persisted?
24+
25+
assert_equal sample.url, 'https://cta-lapp.pages.in2p3.fr/cours/gray_scott_revolutions/grayscott2026/redirect.html?label=sec_gray_scott_webinar_memory_allocation_memory_profiling'
26+
assert_includes sample.description, 'Sometimes memory has become a major problem in applications'
27+
assert_equal sample.end, '2026-02-26 10:30:00 +0000'
28+
assert_equal sample.start, '2026-02-26 09:00:00 +0000'
29+
assert_equal sample.timezone, 'Paris'
30+
assert_includes sample.venue, 'teratec.webex.com'
31+
assert_includes sample.organizer, 'Someone'
32+
end
33+
34+
private
35+
36+
def webmock(url, filename)
37+
file = Rails.root.join('test', 'fixtures', 'files', 'ingestion', filename)
38+
WebMock.stub_request(:get, url).to_return(status: 200, headers: {}, body: file.read)
39+
end
40+
end

0 commit comments

Comments
 (0)