Switch to arrays and content_tag. Improve scraper log formatting

Author: fbacall

app/helpers/materials_helper.rb

@@ -84,24 +84,21 @@ def display_attribute(resource, attribute, show_label: true, title: nil, markdow
                 html_escape(block_given? ? yield(value) : value)
               end
     end
-    string = "<p class=\"#{attribute}#{show_label ? ' no-spacing' : ''}\">"
-    unless value.blank? || value.try(:strip) == 'License Not Specified'
-      string += "<strong class='text-primary'> #{title || resource.class.human_attribute_name(attribute)}: </strong>" if show_label
-      if list
-        string += '<ul>'
-        value.each do |v|
-          string += "<li>#{v}</li>"
+    classes = [attribute]
+    classes << 'no-spacing' if show_label
+    content_tag(:p, class: classes) do
+      unless value.blank? || value.try(:strip) == 'License Not Specified'
+        concat content_tag(:strong, "#{title || resource.class.human_attribute_name(attribute)}: ", class: 'text-primary') if show_label
+        if list
+          concat content_tag(:ul, value.map { |v| content_tag(:li, v) })
+        elsif expandable
+          height_limit = expandable.is_a?(Numeric) ? expandable : nil
+          concat content_tag(:div, value.to_s, class: 'tess-expandable', 'data-height-limit': height_limit ? height_limit : nil)
+        else
+          concat value.to_s
         end
-        string += '</ul>'
-      elsif expandable
-        height_limit = expandable.is_a?(Numeric) ? expandable : nil
-        string += "<div class=\"tess-expandable\"#{" data-height-limit=\"#{height_limit}\"" if height_limit}>" + value.to_s + '</div>'
-      else
-        string += value.to_s
       end
     end
-    string += '</p>'
-    string.html_safe
   end
 
   def display_people(resource, attribute)

lib/ingestors/bioschemas_ingestor.rb

@@ -23,7 +23,7 @@ def read(source_url)
       provider_events = []
       provider_materials = []
       totals = Hash.new(0)
-      no_bioschema_urls = "Bioschemas not found in:\n"
+      no_bioschema_urls = []
       sources.each do |url|
         source = open_url(url)
         output = read_content(source, url: url)
@@ -33,19 +33,19 @@ def read(source_url)
           output[:totals].each do |key, value|
             totals[key] += value
           end
-          no_bioschema_urls += "\n - #{url}" if !source.nil? && output[:totals].values.sum.zero?
+          no_bioschema_urls << url if !source.nil? && output[:totals].values.sum.zero?
         end
       end
 
       if totals.keys.any?
-        bioschemas_summary = "Bioschemas summary:\n"
+        bioschemas_summary = ["Bioschemas summary:\n"]
         totals.each do |type, count|
-          bioschemas_summary += "\n - #{type}: #{count}"
+          bioschemas_summary << " - #{type}: #{count}"
         end
-        @messages << bioschemas_summary
+        @messages << bioschemas_summary.join("\n")
       end
 
-      @messages << no_bioschema_urls
+      @messages << "Bioschemas not found in:\n\n#{no_bioschema_urls.map { |u| " - #{u}" }.join("\n")}" if no_bioschema_urls.any?
 
       deduplicate(provider_events).each do |event_params|
         add_event(event_params)
@@ -113,13 +113,13 @@ def read_content(content, url: nil)
           error = 'A parsing error'
           comment = 'Please check your page contains valid JSON-LD or HTML.'
         end
-        message = "#{error} occurred while reading"
+        message = String.new("#{error} occurred while reading")
         if url.present? && url != 'https://example.com'
-          message += ": #{url} "
+          message << ": #{url} "
         else
-          message += " the source"
+          message << " the source"
         end
-        message += ". #{comment}" if comment
+        message << ". #{comment}" if comment
         @messages << message
       end
 

lib/ingestors/concerns/sitemap_helpers.rb

@@ -41,7 +41,7 @@ def parse_txt_sitemap(url)
       end
 
       def log_sitemap(ext, url, count)
-        @messages << "Parsing .#{ext} sitemap: #{url}\n - #{count} URLs found"
+        @messages << "Parsing .#{ext} sitemap: #{url}\n\n - #{count} URLs found"
       end
     end
   end

lib/ingestors/oai_pmh_ingestor.rb

@@ -94,11 +94,11 @@ def read_oai_rdf(client)
       end
 
       if totals.keys.any?
-        bioschemas_summary = "Bioschemas summary:\n"
+        bioschemas_summary = ["Bioschemas summary:\n"]
         totals.each do |type, count|
-          bioschemas_summary += "\n - #{type}: #{count}"
+          bioschemas_summary << " - #{type}: #{count}"
         end
-        @messages << bioschemas_summary
+        @messages << bioschemas_summary.join("\n")
       end
 
       @bioschemas_manager.deduplicate(provider_events).each do |event_params|

lib/scraper.rb

@@ -104,7 +104,7 @@ def run
 
   def scrape(source, user = get_user, index: 0)
     source_start = Time.now
-    output = ''
+    output = []
     log '', 1
     if source.enabled
       log t('scraper.messages.processing', source: source.content_provider&.title, num: index.to_s), 1
@@ -114,9 +114,9 @@ def scrape(source, user = get_user, index: 0)
     end
     if validate_source(source)
       log t('scraper.messages.valid_source'), 2
-      output += "**Provider:** #{source.content_provider.title}\n\n"
-      output += "<span style='url-wrap'>**URL:** #{source.url}</span>\n\n"
-      output += "**Method:** #{source.ingestor_title}\n\n"
+      output << "**Provider:** #{source.content_provider.title}"
+      output << "<span style='url-wrap'>**URL:** #{source.url}</span>"
+      output << "**Method:** #{source.ingestor_title}"
 
       # get ingestor
       ingestor = Ingestors::IngestorFactory.get_ingestor(source.method)
@@ -127,16 +127,16 @@ def scrape(source, user = get_user, index: 0)
       # read records
       ingestor.read(source.url)
       unless ingestor.messages.blank?
-        output += "\n## Reading\n\n"
-        ingestor.messages.each { |m| output += "#{m}\n" }
+        output << "\n## Reading"
+        output += ingestor.messages
         ingestor.messages.clear
       end
 
       # write resources
       ingestor.write(user, source.content_provider, source: source)
       unless ingestor.messages.blank?
-        output += "\n## Writing\n\n"
-        ingestor.messages.each { |m| output += "#{m}\n" }
+        output << "\n## Writing"
+        output += ingestor.messages
         ingestor.messages.clear
       end
 
@@ -152,14 +152,14 @@ def scrape(source, user = get_user, index: 0)
           ", rejected[#{source.resources_rejected}]", 2
     end
   rescue StandardError => e
-    output += "\n**Failed:** #{e.message}\n\n"
+    output << "\n**Failed:** #{e.message}"
     log "Ingestor: #{ingestor.class} failed with: #{e.message}\t#{e.backtrace[0]}", 2
   ensure
     source.finished_at = Time.now
     run_time = source.finished_at - source_start
-    output += "\n**Finished at:** #{source.finished_at.strftime '%H:%M on %A, %d %B %Y (UTC)'}\n"
-    output += "\n**Run time:** #{run_time.round(2)}s\n"
-    source.log = output
+    output << "\n**Finished at:** #{source.finished_at.strftime '%H:%M on %A, %d %B %Y (UTC)'}"
+    output << "**Run time:** #{run_time.round(2)}s"
+    source.log = output.join("\n\n")
     begin
       # only update enabled sources
       source.save! if source.enabled && !source.is_a?(Scraper::ConfigSource)