skip disabled scrapers and move to bearer auth for eventbrite

Author: DaanVanVugt

config/locales/en.yml

@@ -466,7 +466,8 @@ en:
   scraper:
     messages:
       status: 'Scraper.run: %{status}'
-      processing: 'Processing source: %{source}'
+      processing: 'Processing source %{num}: %{source}'
+      skipped: 'Skipped source %{num}: %{source}'
       processed: 'Sources processed = %{processed}'
       invalid: 'Validation error: %{error_message}'
       valid_source: 'Validation passed!'

lib/ingestors/eventbrite_ingestor.rb

@@ -37,7 +37,7 @@ def process_eventbrite(url)
 
       begin
         # initialise next_page
-        next_page = "#{url}/events/?status=live&token=#{@token}"
+        next_page = "#{url}/events/?status=live"
 
         while next_page
           # execute REST request
@@ -49,7 +49,7 @@ def process_eventbrite(url)
           begin
             if !(pagination.nil? or pagination['has_more_items'].nil? or pagination['page_number'].nil?) && (pagination['has_more_items'])
               page = pagination['page_number'].to_i
-              next_page = "#{url}/events/?page=#{page + 1}&token=#{@token}"
+              next_page = "#{url}/events/?page=#{page + 1}"
             end
           rescue Exception => e
             puts "format next_page failed with: #{e.message}"
@@ -163,7 +163,7 @@ def get_eventbrite_format(id)
 
     def populate_eventbrite_formats
       # get formats from Eventbrite
-      url = "https://www.eventbriteapi.com/v3/formats/?token=#{@token}"
+      url = "https://www.eventbriteapi.com/v3/formats/"
       response = get_json_response url
       # process formats
       response['formats'].each do |format|
@@ -190,7 +190,7 @@ def get_eventbrite_venue(id)
 
     def add_eventbrite_venue(id)
       # get from query and add to cache if found
-      url = "https://www.eventbriteapi.com/v3/venues/#{id}/?token=#{@token}"
+      url = "https://www.eventbriteapi.com/v3/venues/#{id}/"
       venue = get_json_response url
       @eventbrite_objects[:venues][id] = venue unless venue.nil?
     rescue Exception => e
@@ -214,7 +214,7 @@ def get_eventbrite_category(id)
     def populate_eventbrite_categories
       # initialise pagination
       has_more_items = true
-      url = "https://www.eventbriteapi.com/v3/categories/?token=#{@token}"
+      url = "https://www.eventbriteapi.com/v3/categories/"
 
       # query until no more pages
       while has_more_items
@@ -238,7 +238,7 @@ def populate_eventbrite_categories
 
         has_more_items = pagination['has_more_items']
         page_number = pagination['page_number'] + 1
-        url = "https://www.eventbriteapi.com/v3/categories/?page=#{page_number}&token=#{token}"
+        url = "https://www.eventbriteapi.com/v3/categories/?page=#{page_number}"
       end
     rescue Exception => e
       @messages << "get Eventbrite format failed with: #{e.message}"
@@ -270,7 +270,7 @@ def get_eventbrite_subcategory(id, category_id)
     def populate_eventbrite_subcategories(id, category)
       subcategories = nil
       begin
-        url = "#{category['resource_uri']}?token=#{@token}"
+        url = "#{category['resource_uri']}"
         response = get_json_response url
         # updated cached category
         @eventbrite_objects[:categories][id] = response
@@ -297,7 +297,7 @@ def get_eventbrite_organizer(id)
 
     def populate_eventbrite_organizer(id)
       # get from query and add to cache if found
-      url = "https://www.eventbriteapi.com/v3/organizers/#{id}/?token=#{@token}"
+      url = "https://www.eventbriteapi.com/v3/organizers/#{id}/"
       organizer = get_json_response url
       # add to cache
       @eventbrite_objects[:organizers][id] = organizer unless organizer.nil?
@@ -309,7 +309,8 @@ def get_json_response(url, accept_params = 'application/json')
       response = RestClient::Request.new(method: :get,
                                          url: CGI.unescape_html(url),
                                          verify_ssl: false,
-                                         headers: { accept: accept_params }).execute
+                                         headers: { accept: accept_params,
+                                                    authorization: "Bearer #{@token}" }).execute
       # check response
       raise "invalid response code: #{response.code}" unless response.code == 200
 

lib/scraper.rb

@@ -80,7 +80,12 @@ def run
           output = '<ins>**Processing Ingestion Source**</ins><br />'
           processed += 1
           log '', 1
-          log t('scraper.messages.processing', source: processed.to_s), 1
+          if source.enabled
+            log t('scraper.messages.processing', source: source.provider, num: processed.to_s), 1
+          else
+            log t('scraper.messages.skipped', source: source.provider, num: processed.to_s), 1
+            next
+          end
           if validate_source(source)
             log t('scraper.messages.valid_source'), 2
             output.concat '<br />'