Add SIS ignore feature

Author: steve-sullivan

README.md

@@ -25,6 +25,8 @@ UCPath: https://developers.api.berkeley.edu/api/8
 SIS: https://developers.api.berkeley.edu/api/6  
 Additional data is pulled from CalNet via LDAP
 
+Note - the sis_ignore_ids.txt file contains a list of student_ids which will be ignored by the SIS run. This is so employees who take classes (usually UC Extension), don't have their Alma UCPath record over-written by their SIS record.
+
 ### Schedule
 
 * SIS : Monday and Wednesday Mornings 1am

config/settings.yml

@@ -6,7 +6,7 @@ settings:
   upload_host: "upload.lib.berkeley.edu"
   upload_user: "ssullivan"
   last_alma_purge: "2023-06-30"
-  application_version: "1.6.9"
+  application_version: "1.6.10"
 
 # TODO - flesh this out
 # http://docopt.org/

config/sis_ignore_ids.txt

@@ -0,0 +1,2 @@
+# SIS Student_IDs to exclude from student load
+3042087233

lib/sis.rb

@@ -1,3 +1,4 @@
+require 'set'
 require_relative 'sis/api'
 require_relative 'sis/student'
 
@@ -10,24 +11,52 @@ module SIS
   include SIS::API
 
   def run_sis(setup)
+    # Load the Ignore list: see AP-636 for details
+    ignore_list = load_ignored_sis_ids
+
     term_id = setup.term_id || SIS::API.current_term
     as_of_date = SIS::API.as_of_date
+    writer = Alma::XMLWriter.new(setup.xml_path)
 
-    logger.info "Running SIS\nTerm ID: #{term_id}\nRequest Root: #{sis_root} As-of-Date #{as_of_date}"
+    log_run(term_id, as_of_date)
+    process_users(term_id, as_of_date, ignore_list, writer)
+    finalize_output(writer, setup)
+  end
 
-    #----------------------------------------------------------------#
-    # Setup our XML file
-    writer = Alma::XMLWriter.new(setup.xml_path)
+  def log_run(term_id, as_of_date)
+    logger.info "Running SIS\nTerm ID: #{term_id}\nRequest Root: #{sis_root} As-of-Date #{as_of_date}"
+  end
 
-    #----------------------------------------------------------------#
-    # Fetch the entire set of users by term
+  def process_users(term_id, as_of_date, ignore_list, writer)
     raw_users = SIS::API.fetch_by_term(term_id, as_of_date)
+
     raw_users.each do |user|
+      next if ignored_user?(user, ignore_list)
+
       writer.write(SIS::Student.new(user))
     end
+  end
 
+  def finalize_output(writer, setup)
     writer.close
-
     Helpers::FileZip.zipit!(setup.zip_path, setup.xml_path)
   end
+
+  def load_ignored_sis_ids(path = 'config/sis_ignore_ids.txt')
+    return Set.new unless File.exist?(path)
+
+    File.readlines(path, chomp: true)
+      .map(&:strip)
+      .reject { |line| line.empty? || line.start_with?('#') }
+      .to_set
+  end
+
+  def ignored_user?(user, ignore_list)
+    student_id = user['student_id']
+
+    return false unless ignore_list.include?(student_id)
+
+    logger.info "Skipping User #{student_id}: listed in SIS ignore file"
+    true
+  end
 end

spec/lib/sis_spec.rb

@@ -4,37 +4,87 @@
 require 'stub_helper'
 require 'nokogiri'
 
-# rubocop :disable Lint/ConstantDefinitionInBlock, Style/MutableConstant
+# rubocop :disable Lint/ConstantDefinitionInBlock, Style/MutableConstant, Metrics/BlockLength:
 describe SIS do
-  it 'runs sis' do
-    # Make VERSION dynamic:
-    # Note - applicaiton_version is set in config/settings.yml: applicaiton_version
-    expected_version = (ENV['VERSION'] || '1.0.0').to_s
+  describe '#run_sis' do
+    it 'runs sis' do
+      # Make VERSION dynamic:
+      # Note - applicaiton_version is set in config/settings.yml: applicaiton_version
+      expected_version = (ENV['VERSION'] || '1.0.0').to_s
 
-    expected_file = File.read('spec/data/sis/expected_xml_3.xml')
+      expected_file = File.read('spec/data/sis/expected_xml_3.xml')
 
-    allow(Date).to receive(:today).and_return Date.new(2022, 6, 1)
-    expected_file.gsub!(/<!-- VERSION: .+? -->/, "<!-- VERSION: #{expected_version} -->")
+      allow(Date).to receive(:today).and_return Date.new(2022, 6, 1)
+      expected_file.gsub!(/<!-- VERSION: .+? -->/, "<!-- VERSION: #{expected_version} -->")
 
-    term_id = '2222'
-    stub_past_sis_data(term_id, '2022-04-10', 1)
-    stub_past_sis_data(term_id, '2022-04-10', 2)
-    stub_sis_data(term_id, 1)
-    stub_sis_data(term_id, 2)
+      term_id = '2222'
+      stub_past_sis_data(term_id, '2022-04-10', 1)
+      stub_past_sis_data(term_id, '2022-04-10', 2)
+      stub_sis_data(term_id, 1)
+      stub_sis_data(term_id, 2)
+
+      Dir.mktmpdir do |dir|
+        outpath = Pathname.new(dir)
+
+        ARGV = ['--type', 'sis', '-s', '2022-04-10', '--term', '2222', '--outdir', outpath]
+        setup = Helpers::Setup.new
+        SIS.run_sis setup
+
+        expect(File.exist?(setup.zip_path)).to eq(true)
+        expect(File.read(setup.xml_path)).to eq(expected_file)
+      end
+    end
+
+    it 'skips users listed in the SIS ignore file' do
+      allow(Date).to receive(:today).and_return Date.new(2022, 6, 1)
+
+      term_id = '2222'
+      stub_past_sis_data(term_id, '2022-04-10', 1)
+      stub_past_sis_data(term_id, '2022-04-10', 2)
+      stub_sis_data(term_id, 1)
+      stub_sis_data(term_id, 2)
+
+      allow(SIS).to receive(:load_ignored_sis_ids).and_return(Set['10162050'])
 
-    Dir.mktmpdir do |dir|
-      outpath = Pathname.new(dir)
+      Dir.mktmpdir do |dir|
+        outpath = Pathname.new(dir)
 
-      ARGV = ['--type', 'sis', '-s', '2022-04-10', '--term', '2222', '--outdir', outpath]
-      setup = Helpers::Setup.new
-      SIS.run_sis setup
+        ARGV = ['--type', 'sis', '-s', '2022-04-10', '--term', '2222', '--outdir', outpath]
+        setup = Helpers::Setup.new
+
+        SIS.run_sis(setup)
+
+        xml = Nokogiri::XML(File.read(setup.xml_path))
+        primary_ids = xml.xpath('//user/primary_id').map(&:text)
+
+        expect(primary_ids).not_to include('30012345')
+        expect(primary_ids.length).to eq(1)
+      end
+    end
+  end
+
+  describe '.load_ignored_sis_ids' do
+    it 'loads ignored sis ids from a text file' do
+      Dir.mktmpdir do |dir|
+        path = File.join(dir, 'sis_ignore_ids.txt')
+
+        File.write(path, <<~TEXT)
+          # Ignore these users
+          30000001
+
+          30000002
+        TEXT
+
+        expect(SIS.load_ignored_sis_ids(path)).to eq(Set['30000001', '30000002'])
+      end
+    end
 
-      expect(File.exist?(setup.zip_path)).to eq(true)
-      expect(File.read(setup.xml_path)).to eq(expected_file)
+    it 'returns an empty set when the file does not exist' do
+      expect(SIS.load_ignored_sis_ids('does/not/exist.txt')).to eq(Set.new)
     end
   end
 end
-# rubocop :enable Lint/ConstantDefinitionInBlock, Style/MutableConstant
+# rubocop :enable Lint/ConstantDefinitionInBlock, Style/MutableConstant, Metrics/BlockLength:
 
 # rubocop:disable Metrics/BlockLength
 describe SIS::API do