diff --git a/lib/ingestors/cds_videos_ingestor.rb b/lib/ingestors/cds_videos_ingestor.rb new file mode 100644 index 000000000..6c3f3388d --- /dev/null +++ b/lib/ingestors/cds_videos_ingestor.rb @@ -0,0 +1,143 @@ +# frozen_string_literal: true +require 'json' +require 'httparty' +require 'nokogiri' + +module Ingestors + class CdsVideosIngestor < Ingestor # rubocop:disable Style/Documentation + CDS_VIDEOS_RECORD_URL = 'https://videos.cern.ch/record/' + + def self.config + { + key: 'cds videos', + title: 'CDS videos record or search', + category: :materials, + user_agent: 'TeSS CDS Videos ingestor' + } + end + + # Reads from single CDS record URL or search query URL + # For a search query, we need to change fields: page to 1 and size to 100 + # Then read all pages while running `to_material()` + # Else for a single material, go direct with `to_material()` + def read(source_url) + @verbose = false + + api_url = to_api(source_url) + + if source_url.include?('search') + api_url_page_updated = update_url_field_value(api_url, 'page', 1) + api_url_page_size_updated = update_url_field_value(api_url_page_updated, 'size', 100) + data = JSON.parse(open_url(api_url_page_size_updated).read) + add_records_material(data) + else + data = JSON.parse(open_url(api_url).read) + add_material to_material(data) + end + rescue StandardError => e + Rails.logger.error("#{e.class}: read() failed, #{e.message}") + end + + private + + def to_api(url) + uri = URI(url) + parts = uri.path.split('/') # 'example.com/foo/bar' will have path == '/foo/bar', so three parts + + # FROM '/search?{query}' + if parts[1] == 'search' && parts.size == 2 + # TO '/api/records/{query}' + "https://#{uri.host}/api/records?#{uri.query}" + # FROM '/record/{recid}' + elsif parts[1] == 'record' && parts.size == 3 + # TO 'api/record/{recid}' + "https://#{uri.host}/api/#{parts[1]}/#{parts[2]}" + end + end + + def update_url_field_value(url, field, value) + uri = URI.parse(url) + params = URI.decode_www_form(uri.query || '').to_h + params[field] = value.to_s + uri.query = URI.encode_www_form(params) + uri.to_s + end + + def add_records_material(initial_data) + data = initial_data + + while data + hits = data.dig('hits', 'hits') || [] + hits.each { |hit| add_material to_material(hit) } + + next_url = data.dig('links', 'next') + break unless next_url + + response = open_url(next_url)&.read + data = response ? JSON.parse(response) : nil + end + end + + # Sets material hash keys and values and add them to material + def to_material(data) + metadata = data['metadata']&.transform_keys! { |k| k == '_cds' ? 'cds' : k } + + material = OpenStruct.new + material.title = metadata.dig('title', 'title').titleize + material.url = "#{CDS_VIDEOS_RECORD_URL}#{data['id']}" + material.description = get_description(metadata) + material.keywords = metadata['keywords'][0..9]&.map { |k| k['name'] }&.join(', ') || '' + material.licence = metadata.dig('copyright', 'holder') == 'CERN' ? 'other-at' : 'notspecified' + material.status = 'Active' + material.contact = get_contact(metadata) + material.version = metadata['report_number'][0] + material.date_created = metadata['date'] + material.date_published = metadata['publication_date'] + material.authors = get_authors(metadata) + material.contributors = get_contributors(metadata) + material.resource_type = "#{metadata['type'].titleize} – #{metadata['category'].titleize}" + material + end + + def get_description(metadata) + description = metadata['description'].split('
')[0] + "\n\n" + description = description&.<< "Video duration: #{metadata['duration']}\n\n" if metadata['duration'] + description = description + metadata['additional_descriptions']&.map { |k| k['description'] }&.join("\n\n") + "\n\n" || '' if metadata['additional_descriptions'] + description = description&.<< "Related identifier:\n- Is part of: #{metadata['alternate_identifiers'][0]['value']} (URL)\n\n" if metadata['alternate_identifiers']&.any? + description = description&.<< "Copyright: [#{metadata.dig('copyright', 'holder')}](#{metadata.dig('copyright', 'url')}) (#{metadata.dig('copyright', 'year')})" if metadata.dig('copyright', 'holder') + description + end + + def get_authors(metadata) + metadata['contributors'] + &.select { |c| c['role'] == 'Creator' } + &.reject { |c| c['name'].downcase == 'cern' } + &.map { |c| c['name']&.gsub(',', '')&.titleize } || '' + end + + def get_contributors(metadata) + metadata['contributors'] + &.reject { |c| c['role'] == 'Creator' || c['role'] == 'ContactPerson' } + &.reject { |c| c['name'].downcase == 'cern' } + &.map { |c| c['name']&.gsub(',', '')&.titleize } || '' + end + + def get_contact(metadata) + contributors = metadata['contributors'] || [] + + # 1. Check for ContactPerson name + contact_person = contributors.find { |c| c['role'] == 'ContactPerson' } + contact_name = contact_person&.[]('name')&.gsub(',', '')&.titleize + + # 2. Check for Creator email + creator = contributors.find { |c| c['role'] == 'Creator' } + creator_email = creator&.[]('email')&.gsub(',', '')&.titleize + + # 3. Check for any other contributor email + other_contributor = contributors.find { |c| !%w[ContactPerson Creator].include?(c['role']) } + other_email = other_contributor&.[]('email') + + contact_name.presence || creator_email.presence || other_email || '' + end + end +end diff --git a/lib/ingestors/ingestor_factory.rb b/lib/ingestors/ingestor_factory.rb index 8279bd4fa..d4c2d3fac 100644 --- a/lib/ingestors/ingestor_factory.rb +++ b/lib/ingestors/ingestor_factory.rb @@ -15,7 +15,8 @@ def self.ingestors Ingestors::GithubIngestor, Ingestors::EventRSSIngestor, Ingestors::MaterialRSSIngestor, - Ingestors::YoutubeIngestor + Ingestors::YoutubeIngestor, + Ingestors::CdsVideosIngestor ] + taxila_ingestors + llm_ingestors + heptraining_ingestors end diff --git a/test/fixtures/files/ingestion/cdsvideos/cds.ingestor.json b/test/fixtures/files/ingestion/cdsvideos/cds.ingestor.json new file mode 100644 index 000000000..d1ae0e286 --- /dev/null +++ b/test/fixtures/files/ingestion/cdsvideos/cds.ingestor.json @@ -0,0 +1,243 @@ +{ + "created": "2026-03-18T12:40:23.436409+00:00", + "id": "3023615", + "links": { + "self": "https://videos.cern.ch/api/record/3023615" + }, + "metadata": { + "$schema": "https://videos.cern.ch/schemas/records/videos/video/video-v1.0.0.json", + "_access": { + "update": [ + "weblecture-service@cern.ch", + "webcast-team@cern.ch", + "cas-admin@cern.ch" + ] + }, + "_project_id": "20c7e7fb421d49349498594271c5a92b", + "additional_descriptions": [ + { + "description": "CAS - CERN Accelerator School", + "type": "SeriesInformation" + }, + { + "description": "Intensity Limitations in Hadron Beams, 15 - 27 June 2025, Borovets, Bulgaria", + "type": "SeriesInformation" + } + ], + "alternate_identifiers": [ + { + "scheme": "URL", + "value": "https://indico.cern.ch/event/1466612/contributions/6449173/" + } + ], + "category": "LECTURES", + "chapters": [ + { + "seconds": 556, + "timestamp": "00:09:16", + "title": "Slide 2" + }, + { + "seconds": 853, + "timestamp": "00:14:13", + "title": "Slide 3" + }, + { + "seconds": 1025, + "timestamp": "00:17:05", + "title": "Slide 4" + }, + { + "seconds": 1438, + "timestamp": "00:23:58", + "title": "Slide 5" + }, + { + "seconds": 1469, + "timestamp": "00:24:29", + "title": "Slide 6" + }, + { + "seconds": 1534, + "timestamp": "00:25:34", + "title": "Slide 7" + }, + { + "seconds": 1566, + "timestamp": "00:26:06", + "title": "Slide 8" + }, + { + "seconds": 1630, + "timestamp": "00:27:10", + "title": "Slide 9" + }, + { + "seconds": 1648, + "timestamp": "00:27:28", + "title": "Slide 10" + }, + { + "seconds": 1791, + "timestamp": "00:29:51", + "title": "Slide 11" + }, + { + "seconds": 2396, + "timestamp": "00:39:56", + "title": "Slide 12" + }, + { + "seconds": 2453, + "timestamp": "00:40:53", + "title": "Slide 13" + }, + { + "seconds": 2487, + "timestamp": "00:41:27", + "title": "Slide 14" + }, + { + "seconds": 2505, + "timestamp": "00:41:45", + "title": "Slide 15" + }, + { + "seconds": 2679, + "timestamp": "00:44:39", + "title": "Slide 16" + }, + { + "seconds": 2915, + "timestamp": "00:48:35", + "title": "Slide 17" + } + ], + "collections": [ + "Lectures", + "Lectures::Video Lectures", + "Lectures::CERN Accelerator School" + ], + "contributors": [ + { + "affiliations": [ + "CERN" + ], + "email": "cas.news@cern.ch", + "ids": [ + { + "source": "CERN", + "value": "44095" + } + ], + "name": "Noemi Caraban", + "role": "Creator" + }, + { + "name": "cas.news@cern.ch", + "role": "ContactPerson" + }, + { + "email": "no-reply@cern.ch", + "ids": [ + { + "source": "cern", + "value": "9428695" + } + ], + "name": "Dorda, Ulrich", + "role": "Speaker" + } + ], + "copyright": { + "holder": "CERN", + "url": "https://copyright.web.cern.ch/", + "year": "2025" + }, + "date": "2025-06-26", + "description": "After presenting the motivation for an Accelerator Driven System (ADS), the requirements on the accelerator are derived.\nUsing the MYRRHA project as example, the beam optics/dynamics design and operational concept of such an accelerator is discussed.\nFinally, the main technology choices and challenges are presented.\u003Cbr\u003E\u003Cbr\u003E00:00:00 Slide 1\u003Cbr\u003E\n00:09:16 Slide 2\u003Cbr\u003E\n00:14:13 Slide 3\u003Cbr\u003E\n00:17:05 Slide 4\u003Cbr\u003E\n00:23:58 Slide 5\u003Cbr\u003E\n00:24:29 Slide 6\u003Cbr\u003E\n00:25:34 Slide 7\u003Cbr\u003E\n00:26:06 Slide 8\u003Cbr\u003E\n00:27:10 Slide 9\u003Cbr\u003E\n00:27:28 Slide 10\u003Cbr\u003E\n00:29:51 Slide 11\u003Cbr\u003E\n00:39:56 Slide 12\u003Cbr\u003E\n00:40:53 Slide 13\u003Cbr\u003E\n00:41:27 Slide 14\u003Cbr\u003E\n00:41:45 Slide 15\u003Cbr\u003E\n00:44:39 Slide 16\u003Cbr\u003E\n00:48:35 Slide 17\u003Cbr\u003E", + "duration": "00:48:34", + "featured": false, + "keywords": [ + { + "name": "Ulrich Dorda" + }, + { + "name": "Beam cooling" + }, + { + "name": "high-intensity beams" + }, + { + "name": "beam quality" + }, + { + "name": "hadron beams" + }, + { + "name": "cooling techniques" + }, + { + "name": "beam emittance" + }, + { + "name": "stochastic cooling" + }, + { + "name": "electron cooling" + }, + { + "name": "ionization cooling" + }, + { + "name": "Technology" + }, + { + "name": "Topical" + }, + { + "name": "CAS - CERN Accelerator School" + }, + { + "name": "Intensity Limitations in Hadron Beams, 15 - 27 June 2025, Borovets, Bulgaria" + }, + { + "name": "CERN Accelerator School" + }, + { + "name": "1466612c58" + } + ], + "language": "en", + "license": [ + { + "credit": "CERN", + "license": "CERN", + "url": "https://copyright.web.cern" + } + ], + "publication_date": "2026-03-18", + "recid": 3023615, + "related_identifiers": [ + { + "identifier": "1466612", + "relation_type": "IsPartOf", + "scheme": "Indico" + }, + { + "identifier": "https://indico.cern.ch/event/1466612/", + "relation_type": "IsPartOf", + "scheme": "URL" + } + ], + "report_number": [ + "LECTURES-VIDEO-2026-2754-001" + ], + "title": { + "title": "HI for Accelerator Driven Systems" + }, + "type": "VIDEO", + "vr": false + }, + "updated": "2026-03-18T12:40:34.981245+00:00" +} \ No newline at end of file diff --git a/test/unit/ingestors/cds_videos_ingestor_test.rb b/test/unit/ingestors/cds_videos_ingestor_test.rb new file mode 100644 index 000000000..a5e7eb06c --- /dev/null +++ b/test/unit/ingestors/cds_videos_ingestor_test.rb @@ -0,0 +1,125 @@ +# frozen_string_literal: true + +require 'test_helper' + +class CdsVideosIngestorTest < ActiveSupport::TestCase + setup do + @ingestor = Ingestors::CdsVideosIngestor.new + @user = users(:regular_user) + @content_provider = content_providers(:portal_provider) + + # API record + webmock('https://videos.cern.ch/api/record/3023615', 'cdsvideos/cds.ingestor.json') + end + + test 'returns expected ingestor config' do + config = Ingestors::CdsVideosIngestor.config + assert_equal 'cds videos', config[:key] + assert_equal :materials, config[:category] + end + + test 'to_api formats record URL correctly' do + url = 'https://videos.cern.ch/record/3023615' + expected = 'https://videos.cern.ch/api/record/3023615' + assert_equal expected, @ingestor.send(:to_api, url) + end + + test 'to_api formats search URL correctly' do + url = 'https://videos.cern.ch/search?q=test' + expected = 'https://videos.cern.ch/api/records?q=test' + assert_equal expected, @ingestor.send(:to_api, url) + end + + test 'update_url_field_value updates query parameters' do + url = 'https://videos.cern.ch/api/records?q=test&page=2' + updated = @ingestor.send(:update_url_field_value, url, 'page', 1) + assert_equal 'https://videos.cern.ch/api/records?q=test&page=1', updated + end + + test 'should read cds source and map properties correctly' do + @ingestor.read('https://videos.cern.ch/record/3023615') + + assert_equal 1, @ingestor.materials.count + sample = @ingestor.materials.first + + assert_equal 'Hi For Accelerator Driven Systems', sample.title + assert_equal 'https://videos.cern.ch/record/3023615', sample.url + assert_equal 'other-at', sample.licence + assert_equal 'Active', sample.status + assert_equal 'Cas.News@Cern.Ch', sample.contact + assert_equal 'LECTURES-VIDEO-2026-2754-001', sample.version + assert_equal '2025-06-26', sample.date_created + assert_equal '2026-03-18', sample.date_published + assert_equal ['Noemi Caraban'], sample.authors + assert_equal ['Dorda Ulrich'], sample.contributors + assert_equal 'Video – Lectures', sample.resource_type + + expected_keywords = 'Ulrich Dorda, Beam cooling, high-intensity beams, beam quality, hadron beams, cooling techniques, beam emittance, stochastic cooling, electron cooling, ionization cooling' + assert_equal expected_keywords, sample.keywords + + assert_match 'After presenting the motivation for an Accelerator Driven System', sample.description + assert_match 'Video duration: 00:48:34', sample.description + assert_match 'CAS - CERN Accelerator School', sample.description + assert_match 'Is part of: https://indico.cern.ch/event/1466612/contributions/6449173/ (URL)', sample.description + assert_match 'Copyright: [CERN](https://copyright.web.cern.ch/) (2025)', sample.description + end + + test 'std errors when exception is raised' do + mock_logger = Minitest::Mock.new + mock_logger.expect(:error, nil, [/StandardError: read\(\) failed, test failure/]) + + Rails.stub(:logger, mock_logger) do + @ingestor.stub(:open_url, ->(*) { raise StandardError, 'test failure' }) do + @ingestor.read('https://videos.cern.ch/record/3023615') + end + end + + mock_logger.verify + assert true, 'Mock verification passed' + end + +test 'should read search source with pagination across multiple pages' do + search_url = 'https://videos.cern.ch/search?q=physics' + page1_api_url = 'https://videos.cern.ch/api/records?q=physics&page=1&size=100' + page2_api_url = 'https://videos.cern.ch/api/records?q=physics&page=2&size=100' + + record_fixture = JSON.parse(File.read(Rails.root.join('test/fixtures/files/ingestion/cdsvideos/cds.ingestor.json'))) + + page1_payload = { + 'hits' => { 'hits' => [record_fixture] }, + 'links' => { 'next' => page2_api_url } + }.to_json + + page2_payload = { + 'hits' => { 'hits' => [record_fixture] }, + 'links' => {} + }.to_json + + WebMock.stub_request(:get, page1_api_url).to_return(status: 200, body: page1_payload) + WebMock.stub_request(:get, page2_api_url).to_return(status: 200, body: page2_payload) + + @ingestor.read(search_url) + + assert_equal 2, @ingestor.materials.count + assert_equal 'Hi For Accelerator Driven Systems', @ingestor.materials.first.title + end + + test 'should handle search source with empty hits gracefully' do + search_url = 'https://videos.cern.ch/search?q=nonexistent' + api_url = 'https://videos.cern.ch/api/records?q=nonexistent&page=1&size=100' + + empty_payload = { 'hits' => { 'hits' => [] } }.to_json + WebMock.stub_request(:get, api_url).to_return(status: 200, body: empty_payload) + + @ingestor.read(search_url) + + assert_empty @ingestor.materials + end + + private + + def webmock(url, filename) + file = Rails.root.join('test', 'fixtures', 'files', 'ingestion', filename) + WebMock.stub_request(:get, url).to_return(status: 200, headers: {}, body: file.read) + end +end \ No newline at end of file