rdnl scraper

Author: mikesndrs

lib/ingestors/ingestor_factory.rb

@@ -36,6 +36,7 @@ def self.taxila_ingestors
         Ingestors::Taxila::DccIngestor,
         Ingestors::Taxila::SenseIngestor,
         Ingestors::Taxila::VuMaterialIngestor,
+        Ingestors::Taxila::RdnlIngestor,
       ]
     end
 

lib/ingestors/taxila/rdnl_ingestor.rb

@@ -0,0 +1,70 @@
+require 'open-uri'
+require 'csv'
+require 'nokogiri'
+
+module Ingestors
+  module Taxila
+    class RdnlIngestor < Ingestor
+      def self.config
+        {
+          key: 'rdnl_event',
+          title: 'RDNL Events API',
+          category: :events
+        }
+      end
+
+      def read(url)
+        begin
+          process_rdnl(url)
+        rescue Exception => e
+          @messages << "#{self.class.name} failed with: #{e.message}"
+        end
+
+        # finished
+        nil
+      end
+
+      private
+
+      def process_rdnl(url)
+        url = "https://researchdata.nl/agenda/"
+        event_page = Nokogiri::HTML5.parse(open_url(url.to_s, raise: true)).css('main > article > .archive__grid > .column > .archive__content > .column')
+        event_page.each do |event_data|
+          event = OpenStruct.new
+
+          event.url = event_data.css('h2.card__title > a')[0].get_attribute('href')
+          event.title = event_data.css('h2.card__title > a')[0].text.strip
+
+          start_str = event_data.css('dl.meta-list > div > dd')[0].text.strip.split('-')
+          if start_str[1].include?(':')
+            event.start = Time.zone.parse(start_str[0])
+            event.end = Time.zone.parse(start_str[0]).beginning_of_day + Time.zone.parse(start_str[1]).seconds_since_midnight.seconds
+          else
+            event.start = Time.zone.parse(start_str[0])
+            event.end = Time.zone.parse(start_str[1])
+          end
+          event.start = event.start.change(year: event.end.year) if event.start < event.end - 1.year
+          event.start = event.start.change(year: event.end.year - 1) if event.start > event.end
+          if event.start < Time.zone.now - 2.weeks
+            event.start = event.start.change(year: Time.now.year + 1)
+            event.end = event.end.change(year: Time.now.year + 1)
+          end
+
+          event.venue = event_data.css('dl.meta-list > div > dd')[1].text.strip
+          event.source = 'RDNL'
+          event.timezone = 'Amsterdam'
+          event.set_default_times
+          puts event.title
+          puts event.url
+          puts event.start
+          puts event.end
+          puts event.venue
+
+          add_event(event)
+        rescue Exception => e
+          @messages << "Extract event fields failed with: #{e.message}"
+        end
+      end
+    end
+  end
+end

test/unit/ingestors/taxila/rdnl_ingestor_test.rb

@@ -0,0 +1,58 @@
+require 'test_helper'
+
+class RDNLIngestorTest < ActiveSupport::TestCase
+  setup do
+    @user = users(:regular_user)
+    @content_provider = content_providers(:another_portal_provider)
+    mock_ingestions
+    mock_timezone # System time zone should not affect test result
+  end
+
+  teardown do
+    reset_timezone
+  end
+
+  test 'can ingest events from rdnl' do
+    source = @content_provider.sources.build(
+      url: 'https://.knaw.nl/en/agenda/?filter=true&page=',
+      method: 'rdnl',
+      enabled: true
+    )
+
+    ingestor = Ingestors::Taxila::RdnlIngestor.new
+
+    # check event doesn't
+    new_title = 'Essentials 4 Data Support, Feb/Mar 2026'
+    new_url = 'https://researchdata.nl/agenda/essentials-4-data-support-feb-mar-2026/'
+    refute Event.where(title: new_title, url: new_url).any?
+
+    # run task
+    assert_difference 'Event.count', 1 do
+      freeze_time(2025) do
+        VCR.use_cassette('ingestors/rdnl') do
+          ingestor.read(source.url)
+          ingestor.write(@user, @content_provider)
+        end
+      end
+    end
+
+    assert_equal 1, ingestor.events.count
+    assert ingestor.materials.empty?
+    assert_equal 1, ingestor.stats[:events][:added]
+    assert_equal 0, ingestor.stats[:events][:updated]
+    assert_equal 0, ingestor.stats[:events][:rejected]
+
+    # check event does exist
+    event = Event.where(title: new_title, url: new_url).first
+    assert event
+    assert_equal new_title, event.title
+    assert_equal new_url, event.url
+
+    # check other fields
+    assert_equal 'RDNL', event.source
+    assert_equal 'Amsterdam', event.timezone
+    assert_equal 'DANS-KNAW', event.venue
+    assert_equal Time.zone.parse('Thu, 5 Feb 2026 09:00:00.000000000 UTC +00:00'), event.start
+    assert_equal Time.zone.parse('Mon, 26 Mar 2026 17:00:00.000000000 UTC +00:00'), event.end
+  end
+end