diff --git a/lib/ingestors/auto_parsing.rb b/lib/ingestors/auto_parsing.rb index 1b7677cf0..fe1015b06 100644 --- a/lib/ingestors/auto_parsing.rb +++ b/lib/ingestors/auto_parsing.rb @@ -1,23 +1,59 @@ module Ingestors module AutoParsing - def auto_parse(var, description) + def get_mapping(var) @auto_parse_cache ||= {} json_path = File.join(Rails.root, 'lib', 'ingestors', 'auto_parser_mappings', "#{var.to_s}.json") return nil unless File.exist?(json_path) mtime = File.mtime(json_path) cached = @auto_parse_cache[var] - if cached && cached[:mtime] == mtime + if cached && cached[:mtime] == mtime mapping = cached[:mapping] - else + else mapping = JSON.parse(File.read(json_path)) @auto_parse_cache[var] = { mtime: mtime, mapping: mapping } - end + end + mapping + end + + def auto_parse(var, description) + mapping = get_mapping(var) mapping - .select{ |key, val| description&.downcase&.include?(key.to_s.downcase) } + &.select{ |key, val| description&.downcase&.include?(key.to_s.downcase) } &.values &.uniq end + + def handle_auto_parsing(obj) + TeSS::Config.feature['auto_parse_vars'].each do |var| + new_val = auto_parse(var, obj.description) + next if new_val.blank? + + current_val = obj.send(var) if obj.respond_to?(var) + if !obj.respond_to?(var) || current_val.blank? + obj.send("#{var}=", new_val) + end + end + obj + end + + def handle_controlled_vocabulary(obj) + TeSS::Config.feature['controlled_vocabulary_vars'].each do |var| + next unless obj.respond_to?(var) + + mapping = get_mapping(var) + current_val = obj.send(var).map{|x| x.to_s.downcase} + next if current_val.blank? || mapping.blank? + + new_val = mapping + .filter{ |key, val| current_val.include?(key.to_s.downcase) || current_val.include?(val.to_s.downcase) } + .map{ |key, val| val } + .uniq + obj.send("#{var}=", new_val) + end + + obj + end end end diff --git a/lib/ingestors/event_ingestion.rb b/lib/ingestors/event_ingestion.rb index 58985bf8b..737a685e0 100644 --- a/lib/ingestors/event_ingestion.rb +++ b/lib/ingestors/event_ingestion.rb @@ -9,15 +9,8 @@ def add_event(event) c.send(:event_params) event = OpenStruct.new(c.send(:event_params)) end - TeSS::Config.feature['auto_parse_vars'].each do |var| - new_val = auto_parse(var, event.description) - next if new_val.blank? - - current_val = event.send(var) if event.respond_to?(var) - if !event.respond_to?(var) || current_val.blank? - event.send("#{var}=", new_val) - end - end + event = handle_auto_parsing(event) + event = handle_controlled_vocabulary(event) @events << event unless event.nil? end diff --git a/lib/ingestors/material_ingestion.rb b/lib/ingestors/material_ingestion.rb index f4a88db79..81c8fadae 100644 --- a/lib/ingestors/material_ingestion.rb +++ b/lib/ingestors/material_ingestion.rb @@ -9,15 +9,8 @@ def add_material(material) c.send(:material_params) material = OpenStruct.new(c.send(:material_params)) end - TeSS::Config.feature['auto_parse_vars'].each do |var| - new_val = auto_parse(var, material.description) - next if new_val.blank? - - current_val = material.send(var) if material.respond_to?(var) - if !material.respond_to?(var) || current_val.blank? - material.send("#{var}=", new_val) - end - end + material = handle_auto_parsing(material) + material = handle_controlled_vocabulary(material) @materials << material unless material.nil? end end diff --git a/test/unit/ingestors/ingestor_test.rb b/test/unit/ingestors/ingestor_test.rb index 2a56357b5..866fb70bf 100644 --- a/test/unit/ingestors/ingestor_test.rb +++ b/test/unit/ingestors/ingestor_test.rb @@ -131,6 +131,52 @@ class IngestorTest < ActiveSupport::TestCase end end end + + test "handle controlled vocabulary if #{enabled}" do + if enabled == 'enabled' + controlled_vocabulary = ['target_audience', 'keywords'] + else + controlled_vocabulary = [] + end + user = users(:scraper_user) + provider = content_providers(:portal_provider) + @source = Source.create!(url: 'https://somewhere.com/stuff', method: 'bioschemas', + enabled: true, approval_status: 'approved', + content_provider: provider, user: users(:admin)) + ingestor = Ingestors::Ingestor.new + new_event = OpenStruct.new(url: 'https://some-course.net', + title: 'Yet another event', + start: '2021-01-31 13:00:00', + end:'2021-01-31 14:00:00', + description: 'professor, tool criticism and software citation in text', + target_audience: ['nonsense', 'students', 'github beginners'], + keywords: ['nonsense', 'data infrastructure', 'Policy development']) + new_material = OpenStruct.new(url: 'https://some-course.net', + title: 'Yet another course', + description: 'professor, tool criticism and software citation in text', + target_audience: ['nonsense', 'students', 'github beginners'], + keywords: ['nonsense', 'data infrastructure', 'Policy development']) + with_settings({ feature: { controlled_vocabulary_vars: controlled_vocabulary } }) do + ingestor.add_event(new_event) + ingestor.add_material(new_material) + assert_difference('provider.events.count', 1) do + assert_difference('provider.materials.count', 1) do + ingestor.write(user, provider, source: @source) + end + end + end + event = Event.find_by(title: 'Yet another event') + material = Material.find_by(title: 'Yet another course') + [event, material].each do |obj| + if enabled == 'enabled' + assert_equal(['students', 'other'].sort, obj.target_audience.sort) + assert_equal(['Data infrastructure', 'Policy and governance'].sort, obj.keywords.sort) + else + assert_equal(['nonsense', 'students', 'github beginners'], obj.target_audience) + assert_equal(['nonsense', 'data infrastructure', 'Policy development'], obj.keywords) + end + end + end end test 'does not set event language when language and source default language missing' do