Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 47 additions & 11 deletions lib/ingestors/taxila/han_ingestor.rb
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ module Taxila
class HanIngestor < Ingestor
def self.config
{
key: 'han_material',
title: 'HAN Materials API',
category: :materials
key: 'han_event',
title: 'HAN Events API',
category: :events
}
end

Expand All @@ -29,17 +29,53 @@ def read(url)
def process_han(_url)
url = 'https://www.han.nl/studeren/scholing-voor-werkenden/laboratorium/'

material_page = Nokogiri::HTML5.parse(open_url(url.to_s, raise: true)).css('#content > .section--cards-skinny > .section-wrapper > .section__content > .cards-skinny > .cards-skinny-wrapper > .cards-skinny__item > .card-skinny > .card-skinny__content')
material_page.each_with_index do |el, _idx|
material = OpenStruct.new
material.title = el.css('.card-skinny__content__title').first.text
material.url = "https://www.han.nl#{el.css('.card-skinny__content__buttons > .buttons > .buttons__button > a').first.get_attribute('href')}"
material.description = el.css('.card-skinny__content__body').first.text
add_material(material)
event_page = Nokogiri::HTML5.parse(open_url(url.to_s, raise: true)).css('#content > .section--cards-skinny > .section-wrapper > .section__content > .cards-skinny > .cards-skinny-wrapper > .cards-skinny__item > .card-skinny > .card-skinny__content')
event_page.each_with_index do |el, _idx|
event = OpenStruct.new
event.title = el.css('.card-skinny__content__title').first.text
event.url = "https://www.han.nl#{el.css('.card-skinny__content__buttons > .buttons > .buttons__button > a').first.get_attribute('href')}"
event.description = el.css('.card-skinny__content__body').first.text

sleep(1) unless Rails.env.test? and File.exist?('test/vcr_cassettes/ingestors/han.yml')
event_page2 = Nokogiri::HTML5.parse(open_url(event.url.to_s, raise: true))
start_str = event_page2.css("div[class='course-superhero__payoff__content']")[0].css('p > span.course-superhero__payoff__subtitle')[0].text
start_str = convert_months(start_str)
event.start = Time.zone.parse(start_str.split('en')[0].strip)
event.end = event.start
event.set_default_times

course_details = event_page2.css("div[class='course-details__sidebar__item']")[0]
venue_sub_css = course_details.css("span[class='nav-subswitch__title__label__sub']")[0]
venue_super_css = venue_sub_css.parent.css("strong.nav-subswitch__title__label__super")[0]
event.venue = "#{venue_super_css.text} #{venue_sub_css.text}"
event.source = "HAN"
event.timezone = 'Amsterdam'

add_event(event)
rescue Exception => e
@messages << "Extract event fields failed with: #{e.message}"
end
end

def convert_months(my_str)
{
'januari': 'january',
'februari': 'february',
'maart': 'march',
'april': 'april',
'mei': 'may',
'juni': 'june',
'juli': 'july',
'augustus': 'august',
'september': 'september',
'oktober': 'october',
'november': 'november',
'december': 'december',
}.each do |key, value|
my_str = my_str.gsub(key.to_s, value.to_s)
end
my_str
end
end
end
end
end
26 changes: 14 additions & 12 deletions test/unit/ingestors/taxila/han_ingestor_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ class HanIngestorTest < ActiveSupport::TestCase
reset_timezone
end

test 'can ingest materials from han' do
test 'can ingest events from han' do
source = @content_provider.sources.build(
url: 'https://www.han.nl/studeren/scholing-voor-werkenden/laboratorium/',
method: 'han',
Expand All @@ -21,13 +21,13 @@ class HanIngestorTest < ActiveSupport::TestCase

ingestor = Ingestors::Taxila::HanIngestor.new

# check materials don't exist
# check events don't exist
new_title = 'Synthetiseren en Karakteriseren van Moleculen'
new_url = 'https://www.han.nl/opleidingen/module/synthetiseren-karakteriseren-moleculen/'
refute Material.where(title: new_title, url: new_url).any?
refute Event.where(title: new_title, url: new_url).any?

# run task
assert_difference('Material.count', 27) do
assert_difference('Event.count', 15) do
freeze_time(2019) do
VCR.use_cassette('ingestors/han') do
ingestor.read(source.url)
Expand All @@ -37,13 +37,15 @@ class HanIngestorTest < ActiveSupport::TestCase
end

# check event does exist
material = Material.where(title: new_title, url: new_url).first
assert material
assert_equal new_title, material.title
assert_equal new_url, material.url

# check other fields
# assert_equal 'Software Carpentry lessons introduce basic lab skills for research computing. They cover three core topics: the Unix shell, version control with Git, and a programming language (Python or R). ',
# material.description
event = Event.where(title: new_title, url: new_url).first
assert event
assert_equal new_title, event.title
assert_equal new_url, event.url

assert_equal 'HAN', event.source
assert_equal 'Amsterdam', event.timezone
assert_equal 'Nijmegen Laan van Scheut 2', event.venue
assert_equal Time.zone.parse('Mon, 1 Feb 2026 09:00:00.000000000 UTC +00:00'), event.start
assert_equal Time.zone.parse('Mon, 1 Feb 2026 17:00:00.000000000 UTC +00:00'), event.end
end
end
1,441 changes: 1,433 additions & 8 deletions test/vcr_cassettes/ingestors/han.yml

Large diffs are not rendered by default.

Loading