filter ingestor array on controlled_vocabulary list

lib/ingestors/auto_parsing.rb

@@ -1,23 +1,59 @@
 module Ingestors
   module AutoParsing
-    def auto_parse(var, description)
+    def get_mapping(var)
       @auto_parse_cache ||= {}
       json_path = File.join(Rails.root, 'lib', 'ingestors', 'auto_parser_mappings', "#{var.to_s}.json")
       return nil unless File.exist?(json_path)
 
       mtime = File.mtime(json_path)
       cached = @auto_parse_cache[var]
-       if cached && cached[:mtime] == mtime
+        if cached && cached[:mtime] == mtime
         mapping = cached[:mapping]
-       else
+        else
         mapping = JSON.parse(File.read(json_path))
         @auto_parse_cache[var] = { mtime: mtime, mapping: mapping }
-       end
+      end
+      mapping
+    end
+
+    def auto_parse(var, description)
+      mapping = get_mapping(var)
 
       mapping
-        .select{ |key, val| description&.downcase&.include?(key.to_s.downcase) }
+        &.select{ |key, val| description&.downcase&.include?(key.to_s.downcase) }
         &.values
         &.uniq
     end
+
+    def handle_auto_parsing(obj)
+      TeSS::Config.feature['auto_parse_vars'].each do |var|
+        new_val = auto_parse(var, obj.description)
+        next if new_val.blank? 
+
+        current_val = obj.send(var) if obj.respond_to?(var)
+        if !obj.respond_to?(var) || current_val.blank?
+          obj.send("#{var}=", new_val)
+        end
+      end
+      obj
+    end
+
+    def handle_controlled_vocabulary(obj)
+      TeSS::Config.feature['controlled_vocabulary_vars'].each do |var|
+        next unless obj.respond_to?(var)
+
+        mapping = get_mapping(var)
+        current_val = obj.send(var).map{|x| x.to_s.downcase}
+        next if current_val.blank? || mapping.blank?
+
+        new_val = mapping
+                    .filter{ |key, val| current_val.include?(key.to_s.downcase) || current_val.include?(val.to_s.downcase) }
+                    .map{ |key, val| val }
+                    .uniq
+        obj.send("#{var}=", new_val)
+      end
+
+      obj
+    end
   end
 end

lib/ingestors/event_ingestion.rb

@@ -9,15 +9,8 @@ def add_event(event)
         c.send(:event_params)
         event = OpenStruct.new(c.send(:event_params))
       end
-      TeSS::Config.feature['auto_parse_vars'].each do |var|
-        new_val = auto_parse(var, event.description)
-        next if new_val.blank? 
-
-        current_val = event.send(var) if event.respond_to?(var)
-        if !event.respond_to?(var) || current_val.blank?
-          event.send("#{var}=", new_val)
-        end
-      end
+      event = handle_auto_parsing(event)
+      event = handle_controlled_vocabulary(event)
       @events << event unless event.nil?
     end
 

lib/ingestors/material_ingestion.rb

@@ -9,15 +9,8 @@ def add_material(material)
         c.send(:material_params)
         material = OpenStruct.new(c.send(:material_params))
       end
-      TeSS::Config.feature['auto_parse_vars'].each do |var|
-        new_val = auto_parse(var, material.description)
-        next if new_val.blank? 
-
-        current_val = material.send(var) if material.respond_to?(var)
-        if !material.respond_to?(var) || current_val.blank?
-          material.send("#{var}=", new_val)
-        end
-      end
+      material = handle_auto_parsing(material)
+      material = handle_controlled_vocabulary(material)
       @materials << material unless material.nil?
     end
   end

test/unit/ingestors/ingestor_test.rb

@@ -131,6 +131,52 @@ class IngestorTest < ActiveSupport::TestCase
         end
       end
     end
+
+    test "handle controlled vocabulary if #{enabled}" do
+      if enabled == 'enabled'
+        controlled_vocabulary = ['target_audience', 'keywords']
+      else
+        controlled_vocabulary = []
+      end
+      user = users(:scraper_user)
+      provider = content_providers(:portal_provider)
+      @source = Source.create!(url: 'https://somewhere.com/stuff', method: 'bioschemas',
+                              enabled: true, approval_status: 'approved',
+                              content_provider: provider, user: users(:admin))
+      ingestor = Ingestors::Ingestor.new
+      new_event = OpenStruct.new(url: 'https://some-course.net',
+                                  title: 'Yet another event',
+                                  start: '2021-01-31 13:00:00',
+                                  end:'2021-01-31 14:00:00',
+                                  description: 'professor, tool criticism and software citation in text',
+                                  target_audience: ['nonsense', 'students', 'github beginners'],
+                                  keywords: ['nonsense', 'data infrastructure', 'Policy development'])
+      new_material = OpenStruct.new(url: 'https://some-course.net',
+                                  title: 'Yet another course',
+                                  description: 'professor, tool criticism and software citation in text',
+                                  target_audience: ['nonsense', 'students', 'github beginners'],
+                                  keywords: ['nonsense', 'data infrastructure', 'Policy development'])
+      with_settings({ feature: { controlled_vocabulary_vars: controlled_vocabulary } }) do
+        ingestor.add_event(new_event)
+        ingestor.add_material(new_material)
+        assert_difference('provider.events.count', 1) do
+          assert_difference('provider.materials.count', 1) do
+            ingestor.write(user, provider, source: @source)
+          end
+        end
+      end
+      event = Event.find_by(title: 'Yet another event')
+      material = Material.find_by(title: 'Yet another course')
+      [event, material].each do |obj|
+        if enabled == 'enabled'
+          assert_equal(['students', 'other'].sort, obj.target_audience.sort)
+          assert_equal(['Data infrastructure', 'Policy and governance'].sort, obj.keywords.sort)
+        else
+          assert_equal(['nonsense', 'students', 'github beginners'], obj.target_audience)
+          assert_equal(['nonsense', 'data infrastructure', 'Policy development'], obj.keywords)
+        end
+      end
+    end
   end
 
   test 'does not set event language when language and source default language missing' do