Make sure disabled and unapproved sources are skipped

Author: fbacall

lib/scraper.rb

@@ -69,7 +69,7 @@ def run
 
       data_sources = {
         config: @sources.map { |c| Scraper::ConfigSource.new(c.merge(user: user)) },
-        database: Source.find_each
+        database: Source.approved.find_each
       }
 
       data_sources.each do |key, sources|
@@ -81,9 +81,9 @@ def run
           processed += 1
           log '', 1
           if source.enabled
-            log t('scraper.messages.processing', source: source.provider, num: processed.to_s), 1
+            log t('scraper.messages.processing', source: source.content_provider&.title, num: processed.to_s), 1
           else
-            log t('scraper.messages.skipped', source: source.provider, num: processed.to_s), 1
+            log t('scraper.messages.skipped', source: source.content_provider&.title, num: processed.to_s), 1
             next
           end
           if validate_source(source)

test/config/test_ingestion_disabled.yml

@@ -0,0 +1,15 @@
+name: test disabled
+logfile: log/ingestion.test.log
+loglevel: 0
+username: ingestor
+sources:
+  - id: 1
+    provider: 'Another Portal Provider'
+    url: 'https://app.com/events/sitemap.xml'
+    method: ical
+    enabled: true
+  - id: 2
+    provider: 'Another Portal Provider'
+    url: 'https://app.com/events/disabled.xml'
+    method: ical
+    enabled: false

test/fixtures/sources.yml

@@ -94,3 +94,11 @@ enabled_source:
   enabled: true
   user: regular_user
   approval_status: 2
+
+disabled_source:
+  content_provider: portal_provider
+  url: 'https://website.org'
+  method: bioschemas
+  enabled: false
+  user: regular_user
+  approval_status: 2

test/unit/ingestors/scraper_test.rb

@@ -233,6 +233,36 @@ def run
     assert logfile_contains(logfile, 'in `run')
   end
 
+  test 'does not scrape disabled or unapproved sources' do
+    WebMock.stub_request(:get, /https:\/\/app.com\/\d/).to_return(status: 200,
+      body: File.open(Rails.root.join('test', 'fixtures', 'files', 'ingestion', 'events.csv')))
+
+    scraper = Scraper.new(load_scraper_config('test_ingestion_disabled.yml'))
+    provider = content_providers(:goblet)
+    user = users(:admin)
+    unapproved_source = provider.sources.create!(url: 'https://app.com/2', method: 'event_csv', user: user,
+                                                 enabled: true, approval_status: 'not_approved')
+    approval_requested_source = provider.sources.create!(url: 'https://app.com/3', method: 'event_csv', user: user,
+                                                         enabled: true, approval_status: 'requested')
+    User.current_user = user # Admin is required to save approved status
+    enabled_source = provider.sources.create!(url: 'https://app.com/1', method: 'event_csv', user: user,
+                                              enabled: true, approval_status: 'approved')
+    disabled_source = provider.sources.create!(url: 'https://app.com/4', method: 'event_csv', user: user,
+                                               enabled: false, approval_status: 'approved')
+
+    scraper.run
+
+    logfile = scraper.log_file
+    # From Config
+    assert logfile_contains(logfile, "Source URL[https://app.com/events/sitemap.xml]")
+    refute logfile_contains(logfile, "Source URL[https://app.com/events/disabled.xml]")
+    # From Database
+    assert logfile_contains(logfile, "Source URL[#{enabled_source.url}]")
+    refute logfile_contains(logfile, "Source URL[#{disabled_source.url}]")
+    refute logfile_contains(logfile, "Source URL[#{unapproved_source.url}]")
+    refute logfile_contains(logfile, "Source URL[#{approval_requested_source.url}]")
+  end
+
   private
 
   def check_task_finished(logfile)