Filter repositories by declared schema using repository properties

This updates the clone and pull tasks to only apply to repositories
that declare their conformance with a particular OGM schema version
(v1.0 or Aardvark).

This is implemented via a repository custom property, which is set
at the organization level and can be enabled on a per-repository
basis. Unlike repository topics, this value is a controlled vocabulary
and is unique to the OGM organization.

This functionality replaces the "denylist" that was a hardcoded list
of repositories that shouldn't be harvested (e.g. because they were
tools or didn't contain metadata). In this version, providers can
opt-in to harvesting by adding the relevant schema property.

After this change, you no longer need to clone repositories that
don't match the schema version you are using, which saves disk
space and processing time.

Author: thatbudakguy

.rubocop_todo.yml

@@ -41,7 +41,7 @@ Metrics/CyclomaticComplexity:
 # Offense count: 13
 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
 Metrics/MethodLength:
-  Max: 21
+  Max: 25
 
 # Offense count: 2
 # Configuration parameters: AllowedMethods, AllowedPatterns.

lib/geo_combine/harvester.rb

@@ -11,19 +11,6 @@ module GeoCombine
   class Harvester
     attr_reader :ogm_path, :schema_version
 
-    # Non-metadata repositories that shouldn't be harvested
-    def self.denylist
-      [
-        'GeoCombine',
-        'aardvark',
-        'metadata-issues',
-        'ogm_utils-python',
-        'opengeometadata.github.io',
-        'opengeometadata-rails',
-        'gbl-1_to_aardvark'
-      ]
-    end
-
     # GitHub API endpoint for OpenGeoMetadata repositories
     def self.ogm_api_uri
       URI('https://api.github.com/orgs/opengeometadata/repos?per_page=1000')
@@ -53,9 +40,16 @@ def docs_to_index
 
         doc = JSON.parse(File.read(path))
         [doc].flatten.each do |record|
-          # skip indexing if this record has a different schema version than what we want
           record_schema = record['gbl_mdVersion_s'] || record['geoblacklight_version']
           record_id = record['layer_slug_s'] || record['dc_identifier_s']
+
+          # skip indexing if no identifiable schema version
+          unless record_schema
+            @logger.debug "skipping #{record_id || path}; no schema version declared in record"
+            next
+          end
+
+          # skip indexing if this record has a different schema version than what we want
           if record_schema != @schema_version
             @logger.debug "skipping #{record_id}; schema version #{record_schema} doesn't match #{@schema_version}"
             next
@@ -87,19 +81,20 @@ def pull_all
     end
 
     # Clone a repository via git
-    # If the repository already exists, skip it.
+    # Return the name of the repository cloned, or nil if skipped
     def clone(repo)
       repo_path = File.join(@ogm_path, repo)
       repo_info = repository_info(repo)
       repo_url = "https://github.com/OpenGeoMetadata/#{repo}.git"
-
-      # Skip if exists; warn if archived or empty
-      if File.directory? repo_path
-        @logger.warn "skipping clone to #{repo_path}; directory exists"
-        return nil
+      repo_schemas = Array(repo_info.dig('custom_properties', 'supported_schemas'))
+
+      # Skip if exists, archived, empty, or different schema
+      return @logger.warn "skipping clone to #{repo_path}; directory exists" if File.directory? repo_path
+      return @logger.warn "repository is archived: #{repo_url}" if repo_info['archived']
+      return @logger.warn "repository is empty: #{repo_url}" if repo_info['size'].zero?
+      unless repo_schemas.include? @schema_version
+        return @logger.warn "repository #{repo_url} clone to #{repo_path}; repository properties don't include schema version #{@schema_version} (found #{repo_schemas.join(', ')})"
       end
-      @logger.warn "repository is archived: #{repo_url}" if repo_info['archived']
-      @logger.warn "repository is empty: #{repo_url}" if repo_info['size'].zero?
 
       Git.clone(repo_url, nil, path: ogm_path, depth: 1)
       @logger.info "cloned #{repo_url} to #{repo_path}"
@@ -119,10 +114,10 @@ def clone_all
     # List of repository names to harvest
     def repositories
       @repositories ||= JSON.parse(Net::HTTP.get(self.class.ogm_api_uri))
+                            .filter { |repo| Array(repo.dig('custom_properties', 'supported_schemas')).include? @schema_version }
                             .filter { |repo| repo['size'].positive? }
                             .reject { |repo| repo['archived'] }
                             .map { |repo| repo['name'] }
-                            .reject { |name| self.class.denylist.include? name }
     end
 
     def repository_info(repo_name)

spec/lib/geo_combine/harvester_spec.rb

@@ -5,7 +5,7 @@
 require 'spec_helper'
 
 RSpec.describe GeoCombine::Harvester do
-  subject(:harvester) { described_class.new(ogm_path: 'spec/fixtures/indexing', schema_version: '1.0') }
+  subject(:harvester) { described_class.new(ogm_path: 'spec/fixtures/indexing', logger: logger) }
 
   let(:logger) { instance_double(Logger, warn: nil, info: nil, error: nil, debug: nil) }
   let(:repo_name) { 'my-institution' }
@@ -14,11 +14,12 @@
   let(:stub_repo) { instance_double(Git::Base) }
   let(:stub_gh_api) do
     [
-      { name: repo_name, size: 100 },
-      { name: 'another-institution', size: 100 },
-      { name: 'outdated-institution', size: 100, archived: true }, # archived
-      { name: 'aardvark', size: 300 },                             # on denylist
-      { name: 'empty', size: 0 }                                   # no data
+      { name: repo_name, size: 100, custom_properties: { supported_schemas: ['Aardvark'] } },
+      { name: 'another-institution', size: 100, custom_properties: { supported_schemas: ['Aardvark', '1.0'] } }, # multiple schemas
+      { name: 'v1-institution', size: 300, custom_properties: { supported_schemas: ['1.0'] } }, # schema mismatch
+      { name: 'outdated-institution', size: 100, custom_properties: { supported_schemas: ['Aardvark'] }, archived: true }, # archived
+      { name: 'empty', size: 0, custom_properties: { supported_schemas: ['Aardvark'] } }, # no data
+      { name: 'tool', size: 50 } # not a metadata repository
     ]
   end
 
@@ -42,15 +43,15 @@
   describe '#docs_to_index' do
     it 'yields each JSON record with its path, skipping layers.JSON' do
       expect { |b| harvester.docs_to_index(&b) }.to yield_successive_args(
-        [JSON.parse(File.read('spec/fixtures/indexing/basic_geoblacklight.json')), 'spec/fixtures/indexing/basic_geoblacklight.json'],
-        [JSON.parse(File.read('spec/fixtures/indexing/geoblacklight.json')), 'spec/fixtures/indexing/geoblacklight.json']
+        [JSON.parse(File.read('spec/fixtures/indexing/aardvark.json')), 'spec/fixtures/indexing/aardvark.json']
       )
     end
 
-    it 'skips records with a different schema version' do
-      harvester = described_class.new(ogm_path: 'spec/fixtures/indexing/', schema_version: 'Aardvark', logger:)
+    it 'can yield JSON records for a different schema version' do
+      harvester = described_class.new(ogm_path: 'spec/fixtures/indexing/', schema_version: '1.0', logger:)
       expect { |b| harvester.docs_to_index(&b) }.to yield_successive_args(
-        [JSON.parse(File.read('spec/fixtures/indexing/aardvark.json')), 'spec/fixtures/indexing/aardvark.json']
+        [JSON.parse(File.read('spec/fixtures/indexing/basic_geoblacklight.json')), 'spec/fixtures/indexing/basic_geoblacklight.json'],
+        [JSON.parse(File.read('spec/fixtures/indexing/geoblacklight.json')), 'spec/fixtures/indexing/geoblacklight.json']
       )
     end
   end
@@ -79,15 +80,20 @@
       expect(harvester.pull_all).to eq(%w[my-institution another-institution])
     end
 
-    it 'skips repositories in the denylist' do
+    it 'skips repositories with no schema declared' do
       harvester.pull_all
-      expect(Git).not_to have_received(:open).with('https://github.com/OpenGeoMetadata/aardvark.git')
+      expect(Git).not_to have_received(:open).with('https://github.com/OpenGeoMetadata/tool.git')
     end
 
     it 'skips archived repositories' do
       harvester.pull_all
       expect(Git).not_to have_received(:open).with('https://github.com/OpenGeoMetadata/outdated-institution.git')
     end
+
+    it 'skips repositories with no data' do
+      harvester.pull_all
+      expect(Git).not_to have_received(:open).with('https://github.com/OpenGeoMetadata/empty.git')
+    end
   end
 
   describe '#clone' do
@@ -115,9 +121,19 @@
       expect(Git).to have_received(:clone).exactly(2).times
     end
 
-    it 'skips repositories in the denylist' do
+    it 'skips repositories with no schema declared' do
+      harvester.clone_all
+      expect(Git).not_to have_received(:clone).with('https://github.com/OpenGeoMetadata/tool.git')
+    end
+
+    it 'skips archived repositories' do
+      harvester.clone_all
+      expect(Git).not_to have_received(:clone).with('https://github.com/OpenGeoMetadata/outdated-institution.git')
+    end
+
+    it 'skips repositories with no data' do
       harvester.clone_all
-      expect(Git).not_to have_received(:clone).with('https://github.com/OpenGeoMetadata/aardvark.git')
+      expect(Git).not_to have_received(:clone).with('https://github.com/OpenGeoMetadata/empty.git')
     end
 
     it 'returns the names of repositories cloned' do