Google sheet ingestor for events

Author: fbacall

lib/ingestors/event_csv_ingestor.rb

@@ -3,7 +3,7 @@
 
 module Ingestors
   class EventCsvIngestor < Ingestor
-    include CsvIngestion
+    include SpreadsheetIngestion
 
     def self.config
       {
@@ -16,6 +16,9 @@ def self.config
     def read(url)
       # parse csv file to table
       begin
+        # Google spreadsheet convertor
+        url = gsheet_to_csv(url)
+
         # parse csv as table
         web_contents = open_url(url).read
         table = CSV.parse(web_contents, headers: true)

lib/ingestors/material_csv_ingestor.rb

@@ -4,7 +4,7 @@
 
 module Ingestors
   class MaterialCsvIngestor < Ingestor
-    include CsvIngestion
+    include SpreadsheetIngestion
 
     def self.config
       {
@@ -68,16 +68,5 @@ def read(url)
     def process_competency(row, header)
       row[header].nil? ? 'notspecified' : row[header]
     end
-
-    # if url is a raw google spreadsheet
-    # it returns the Google spreadsheet CSV export
-    # else it returns the url
-    def gsheet_to_csv(url)
-      return url unless url.include? 'docs.google.com/spreadsheets/d/'
-
-      spreadsheet_id = url.partition('d/').last.partition('/').first
-      gid = CGI.parse(URI.parse(url).query)['gid']&.first
-      "https://docs.google.com/spreadsheets/d/#{spreadsheet_id}/export?gid=#{gid}&exportFormat=csv"
-    end
   end
 end

lib/ingestors/csv_ingestion.rb lib/ingestors/spreadsheet_ingestion.rblib/ingestors/csv_ingestion.rb renamed to lib/ingestors/spreadsheet_ingestion.rb

@@ -1,5 +1,5 @@
 module Ingestors
-  module CsvIngestion
+  module SpreadsheetIngestion
     def process_url(row, header)
       row[header].to_s.lstrip unless row[header].nil?
     end
@@ -21,5 +21,16 @@ def process_array(row, header)
     def get_column(row, header)
       row[header].to_s.lstrip unless row[header].nil?
     end
+
+    # if url is a raw google spreadsheet
+    # it returns the Google spreadsheet CSV export
+    # else it returns the url
+    def gsheet_to_csv(url)
+      return url unless url.include? 'docs.google.com/spreadsheets/d/'
+
+      spreadsheet_id = url.partition('d/').last.partition('/').first
+      gid = CGI.parse(URI.parse(url).query)['gid']&.first
+      "https://docs.google.com/spreadsheets/d/#{spreadsheet_id}/export?gid=#{gid}&exportFormat=csv"
+    end
   end
 end

test/unit/ingestors/event_csv_ingestor_test.rb

@@ -95,6 +95,23 @@ def run
                  event.tech_requirements
   end
 
+  test 'should change url to google sheet export if needed' do
+    ingestor = Ingestors::EventCsvIngestor.new
+
+    url_random = 'https://this-might-not-be-reachable-at-all.com'
+    url_not_spreadsheet = 'https://google.com'
+    url_spreadsheet = 'https://docs.google.com/spreadsheets/d/abcde/edit?gid=12345#gid=12345'
+    url_export_spreadsheet = 'https://docs.google.com/spreadsheets/d/abcde/export?gid=12345&exportFormat=csv'
+
+    # doesn't change url, because not google spreadsheet
+    assert_equal ingestor.send(:gsheet_to_csv, url_random), url_random
+    assert_equal ingestor.send(:gsheet_to_csv, url_not_spreadsheet), url_not_spreadsheet
+
+    # do change url, because google spreadsheet
+    assert_equal ingestor.send(:gsheet_to_csv, url_spreadsheet), url_export_spreadsheet
+    assert_not_equal url_spreadsheet, url_export_spreadsheet
+  end
+
   private
 
   def check_array(collection, values, exclusions = [])