Merge pull request #1153 from DaanVanVugt/bugfix/taxila_repair

repair broken taxila scrapers

Author: fbacall

lib/ingestors/taxila/dans_ingestor.rb

@@ -54,7 +54,7 @@ def process_dans(url)
               event.keywords << value unless value.nil?
             end
 
-            event.description = data.css("p[class='dmach-acf-value dmach-acf-video-container']")[0].children.to_s
+            event.description = data.css("p.dmach-acf-value")[0].text
 
             event.url = data.css("a[id$='_link']")[0]['href'].to_s
 

lib/ingestors/taxila/dcc_ingestor.rb

@@ -27,15 +27,14 @@ def read(url)
       private
 
       def process_dcc(url)
-        # event_page = Nokogiri::HTML5.parse(open_url(url.to_s, raise: true)).css('main > .archive__content.grid > .column.span-4-sm.span-4-md.span-6-lg')
         event_page = Nokogiri::HTML5.parse(open_url(url.to_s, raise: true)).css('main > article > .archive__grid > .column > .archive__content > .column')
         event_page.each do |event_data|
           event = OpenStruct.new
 
-          event.url = event_data.css('h2.post-item__title > a')[0].get_attribute('href')
-          event.title = event_data.css('h2.post-item__title > a')[0].text.strip
+          event.url = event_data.css('h2.card__title > a')[0].get_attribute('href')
+          event.title = event_data.css('h2.card__title > a')[0].text.strip
 
-          start_str = event_data.css('ul.post-item__meta > li')[0].text.strip.split('—')
+          start_str = event_data.css('dl.meta-list > div > dd')[0].text.strip.split('-')
           if start_str[1].include?(':')
             event.start = Time.zone.parse(start_str[0])
             event.end = Time.zone.parse(start_str[0]).beginning_of_day + Time.zone.parse(start_str[1]).seconds_since_midnight.seconds
@@ -48,8 +47,7 @@ def process_dcc(url)
             event.end = event.end.change(year: Time.now.year + 1)
           end
 
-          event.venue = event_data.css('ul.post-item__meta > li')[1].text.strip
-
+          event.venue = event_data.css('dl.meta-list > div > dd')[1].text.strip
           event.source = 'DCC'
           event.timezone = 'Amsterdam'
           event.set_default_times

lib/ingestors/taxila/tdcc_ingestor.rb

@@ -35,7 +35,7 @@ def process_tdcc(url)
           event.url = event_data.css("h2.card__title a")[0].get_attribute('href').strip
           event.venue = event_data.css("dl.meta-list svg.icon--marker")[0]&.parent&.text&.strip || 'Online'
           time_str = event_data.css("dl.meta-list svg.icon--calendar")[0].parent.text.strip
-          split_time_str = time_str.split(' — ')
+          split_time_str = time_str.split(' - ')
           event.start = Time.zone.parse(split_time_str[0])
           if split_time_str[1].split(' ').length == 1
             a = split_time_str[0].split(' ')

test/unit/ingestors/taxila/dans_ingestor_test.rb

@@ -22,23 +22,23 @@ class DansIngestorTest < ActiveSupport::TestCase
     ingestor = Ingestors::Taxila::DansIngestor.new
 
     # check event doesn't
-    new_title = 'Open Hour SSH: live Q&A on Monday'
-    new_url = 'https://dans.knaw.nl/en/agenda/open-hour-ssh-live-qa-on-monday-2/'
+    new_title = 'Open Science Festival 2025'
+    new_url = 'https://opensciencefestival.nl/en/register-as-a-participant'
     refute Event.where(title: new_title, url: new_url).any?
 
     # run task
-    assert_difference 'Event.count', 6 do
-      freeze_time(2019) do
+    assert_difference 'Event.count', 11 do
+      freeze_time(2025) do
         VCR.use_cassette('ingestors/dans') do
           ingestor.read(source.url)
           ingestor.write(@user, @content_provider)
         end
       end
     end
 
-    assert_equal 6, ingestor.events.count
+    assert_equal 11, ingestor.events.count
     assert ingestor.materials.empty?
-    assert_equal 6, ingestor.stats[:events][:added]
+    assert_equal 11, ingestor.stats[:events][:added]
     assert_equal 0, ingestor.stats[:events][:updated]
     assert_equal 0, ingestor.stats[:events][:rejected]
 
@@ -51,8 +51,8 @@ class DansIngestorTest < ActiveSupport::TestCase
     # check other fields
     assert_equal 'DANS', event.source
     assert_equal 'Amsterdam', event.timezone
-    assert_equal ['Social Sciences and Humanities', 'Training &amp; Outreach', 'Consultancy'], event.keywords
-    assert_equal Time.zone.parse('Mon, 13 Feb 2023 09:00:00.000000000 UTC +00:00'), event.start
-    assert_equal Time.zone.parse('Mon, 13 Feb 2023 17:00:00.000000000 UTC +00:00'), event.end
+    assert_equal ['FAIR and Open data'], event.keywords
+    assert_equal Time.zone.parse('Mon, 24 Oct 2025 09:00:00.000000000 UTC +00:00'), event.start
+    assert_equal Time.zone.parse('Mon, 24 Oct 2025 17:00:00.000000000 UTC +00:00'), event.end
   end
 end

test/unit/ingestors/taxila/dcc_ingestor_test.rb

@@ -22,13 +22,13 @@ class DccIngestorTest < ActiveSupport::TestCase
     ingestor = Ingestors::Taxila::DccIngestor.new
 
     # check event doesn't
-    new_title = 'Training FAIR data management'
-    new_url = 'https://dcc-po.nl/agenda/training-fair-data-management/'
+    new_title = 'Training Beïnvloedings-vaardigheden'
+    new_url = 'https://dcc-po.nl/agenda/training-beinvloedingsvaardigheden/'
     refute Event.where(title: new_title, url: new_url).any?
 
     # run task
     assert_difference 'Event.count', 2 do
-      freeze_time(2019) do
+      freeze_time(2025) do
         VCR.use_cassette('ingestors/dcc') do
           ingestor.read(source.url)
           ingestor.write(@user, @content_provider)
@@ -51,8 +51,8 @@ class DccIngestorTest < ActiveSupport::TestCase
     # check other fields
     assert_equal 'DCC', event.source
     assert_equal 'Amsterdam', event.timezone
-    assert_equal Time.zone.parse('Fri, 30 Sep 2019 09:00:00.000000000 UTC +00:00'), event.start
-    assert_equal Time.zone.parse('Sat, 14 Oct 2019 17:00:00.000000000 UTC +00:00'), event.end
-    assert_equal 'Domstad, Utrecht', event.venue
+    assert_equal Time.zone.parse('Fri, 18 Sep 2025 09:00:00.000000000 UTC +00:00'), event.start
+    assert_equal Time.zone.parse('Sat, 10 Oct 2025 17:00:00.000000000 UTC +00:00'), event.end
+    assert_equal 'Utrecht, SURF en HU', event.venue
   end
 end

test/unit/ingestors/taxila/tdcc_ingestor_test.rb

@@ -22,23 +22,23 @@ class TdccIngestorTest < ActiveSupport::TestCase
     ingestor = Ingestors::Taxila::TdccIngestor.new
 
     # check event doesn't
-    new_title = "Project Idea Generator - An Unconference by the Thematic DCC Social Sciences and Humanities"
-    new_url = 'https://tdcc.nl/evenementen/project-idea-generator-an-unconference-by-the-thematic-dcc-social-sciences-humanities/'
+    new_title = "TDCC-NES Governing Board meeting, October 2025"
+    new_url = 'https://tdcc.nl/evenementen/tdcc-nes-governing-board-meeting-october-2025/'
     refute Event.where(title: new_title, url: new_url).any?
 
     # run task
-    assert_difference 'Event.count', 8 do
-      freeze_time(2023) do
+    assert_difference 'Event.count', 2 do
+      freeze_time(2025) do
         VCR.use_cassette("ingestors/tdcc") do
           ingestor.read(source.url)
           ingestor.write(@user, @content_provider)
         end
       end
     end
 
-    assert_equal 8, ingestor.events.count
+    assert_equal 2, ingestor.events.count
     assert ingestor.materials.empty?
-    assert_equal 8, ingestor.stats[:events][:added]
+    assert_equal 2, ingestor.stats[:events][:added]
     assert_equal 0, ingestor.stats[:events][:updated]
     assert_equal 0, ingestor.stats[:events][:rejected]
 
@@ -51,8 +51,8 @@ class TdccIngestorTest < ActiveSupport::TestCase
     # check other fields
     assert_equal 'TDCC', event.source
     assert_equal 'Amsterdam', event.timezone
-    assert_equal Time.zone.parse('Thu, 2 Apr 2023 13:00:00.000000000 UTC +00:00'), event.start
-    assert_equal Time.zone.parse('Thu, 2 Apr 2023 17:00:00.000000000 UTC +00:00'), event.end
-    assert_equal 'Het Trippenhuis, Amsterdam', event.venue
+    assert_equal Time.zone.parse('Mon, 9 Oct 2025 09:00:00.000000000 UTC +00:00'), event.start
+    assert_equal Time.zone.parse('Mon, 9 Oct 2025 17:00:00.000000000 UTC +00:00'), event.end
+    assert_equal 'Social Impact Factory', event.venue
   end
 end