copilot comments

Author: mikesndrs

lib/ingestors/auto_parsing.rb

@@ -0,0 +1,23 @@
+module Ingestors
+  module AutoParsing
+    def auto_parse(var, description)
+      @auto_parse_cache ||= {}
+      json_path = File.join(Rails.root, 'lib', 'ingestors', 'auto_parser_mappings', "#{var.to_s}.json")
+      return nil unless File.exist?(json_path)
+
+      mtime = File.mtime(json_path)
+      cached = @auto_parse_cache[var]
+       if cached && cached[:mtime] == mtime
+        mapping = cached[:mapping]
+       else
+        mapping = JSON.parse(File.read(json_path))
+        @auto_parse_cache[var] = { mtime: mtime, mapping: mapping }
+       end
+       
+      mapping
+        .select{ |key, val| description&.downcase&.include?(key.to_s.downcase) }
+        &.values
+        &.uniq
+    end
+  end
+end

lib/ingestors/event_ingestion.rb

@@ -1,5 +1,7 @@
 module Ingestors
   module EventIngestion
+    include AutoParsing
+
     def add_event(event)
       if event.is_a?(Hash)
         c = EventsController.new
@@ -9,7 +11,12 @@ def add_event(event)
       end
       TeSS::Config.feature['auto_parse_vars'].each do |var|
         new_val = auto_parse(var, event.description)
-        event.send("#{var}=", new_val)
+        next if new_val.blank? 
+
+        current_val = event.send(var) if event.respond_to?(var)
+        if !event.respond_to?(var) || current_val.blank?
+          event.send("#{var}=", new_val)
+        end
       end
       @events << event unless event.nil?
     end
@@ -26,19 +33,6 @@ def convert_location(input)
       input
     end
 
-    def auto_parse(var, description)
-      json_path = File.join(Rails.root, 'lib', 'ingestors', 'auto_parser_mappings', "#{var.to_s}.json")
-      res = nil
-      if File.exist?(json_path)
-        mapping = JSON.parse(File.read(json_path))
-        res = mapping
-          .select{ |key, val| description.downcase.include?(key.to_s.downcase) }
-          .values
-          .uniq
-      end
-      res
-    end
-
     def parse_dates(input, timezone = nil)
       Time.use_zone(timezone) do
         # try to split on obvious interval markers

lib/ingestors/material_ingestion.rb

@@ -1,5 +1,7 @@
 module Ingestors
   module MaterialIngestion
+    include AutoParsing
+
     def add_material(material)
       if material.is_a?(Hash)
         c = MaterialsController.new
@@ -8,23 +10,15 @@ def add_material(material)
         material = OpenStruct.new(c.send(:material_params))
       end
       TeSS::Config.feature['auto_parse_vars'].each do |var|
-        new_val = auto_parse(var, event.description)
-        event.send("#{var}=", new_val)
-      end
-      @materials << material unless material.nil?
-    end
+        new_val = auto_parse(var, material.description)
+        next if new_val.blank? 
 
-    def auto_parse(var, description)
-      json_path = File.join(Rails.root, 'lib', 'ingestors', 'auto_parser_mappings', "#{var.to_s}.json")
-      res = nil
-      if File.exist?(json_path)
-        mapping = JSON.parse(File.read(json_path))
-        res = mapping
-          .select{ |key, val| description.downcase.include?(key.to_s.downcase) }
-          .values
-          .uniq
+        current_val = material.send(var) if material.respond_to?(var)
+        if !material.respond_to?(var) || current_val.blank?
+          material.send("#{var}=", new_val)
+        end
       end
-      res
+      @materials << material unless material.nil?
     end
   end
 end

test/unit/ingestors/ingestor_test.rb

@@ -103,23 +103,32 @@ class IngestorTest < ActiveSupport::TestCase
                               content_provider: provider, user: users(:admin))
       ingestor = Ingestors::Ingestor.new
       new_event = OpenStruct.new(url: 'https://some-course.net',
-                                  title: 'Yet another course',
+                                  title: 'Yet another event',
                                   start: '2021-01-31 13:00:00',
                                   end:'2021-01-31 14:00:00',
                                   description: 'professor, tool criticism and software citation in text')
+      new_material = OpenStruct.new(url: 'https://some-course.net',
+                                  title: 'Yet another course',
+                                  description: 'professor, tool criticism and software citation in text')
       with_settings({ feature: { auto_parse_vars: auto_parse_vars } }) do
         ingestor.add_event(new_event)
+        ingestor.add_material(new_material)
         assert_difference('provider.events.count', 1) do
-          ingestor.write(user, provider, source: @source)
+          assert_difference('provider.materials.count', 1) do
+            ingestor.write(user, provider, source: @source)
+          end
         end
       end
-      event = Event.find_by(title: 'Yet another course')
-      if enabled == 'enabled'
-        assert_equal(['researchers'], event.target_audience)
-        assert_equal(['Research software management', 'Data infrastructure'], event.keywords)
-      else
-        assert_equal([], event.target_audience)
-        assert_equal([], event.keywords)
+      event = Event.find_by(title: 'Yet another event')
+      material = Material.find_by(title: 'Yet another course')
+      [event, material].each do |obj|
+        if enabled == 'enabled'
+          assert_equal(['researchers'].sort, obj.target_audience.sort)
+          assert_equal(['Research software management', 'Data infrastructure'].sort, obj.keywords.sort)
+        else
+          assert_equal([], obj.target_audience)
+          assert_equal([], obj.keywords)
+        end
       end
     end
   end