feat: use rspec retry to deflake ci

Gemfile

@@ -98,6 +98,7 @@ end
 group :test do
   gem "brakeman" # security inspection
   gem "capybara"
+  gem "rspec-retry" # for retrying flaky tests
   gem "capybara-screenshot"
   gem "database_cleaner-active_record"
   gem "docx"

Gemfile.lock

@@ -472,6 +472,8 @@ GEM
       rspec-expectations (~> 3.13)
       rspec-mocks (~> 3.13)
       rspec-support (~> 3.13)
+    rspec-retry (0.6.2)
+      rspec-core (> 3.3)
     rspec-support (3.13.2)
     rspec_junit_formatter (0.6.0)
       rspec-core (>= 2, < 4, != 2.12.0)
@@ -695,6 +697,7 @@ DEPENDENCIES
   request_store
   rexml
   rspec-rails
+  rspec-retry
   rspec_junit_formatter
   rswag-api
   rswag-specs

app/javascript/controllers/multiple_select_controller.js

@@ -84,8 +84,8 @@ export default class extends Controller {
           label: '<i class="lni lni-cross-circle"></i>'
         },
         checkbox_options: {
-          checkedClassNames: ['form-check-input'],
-          uncheckedClassNames: ['form-check-input']
+          checkedClassNames: ['form-check-input', 'form-check-input--checked'],
+          uncheckedClassNames: ['form-check-input', 'form-check-input--unchecked']
         }
       },
       options: dropdownOptions,

spec/spec_helper.rb

@@ -12,30 +12,30 @@
 # the additional setup, and require it from the spec files that actually need
 # it.
 
-# if ENV["RUN_SIMPLECOV"]
-#   require "simplecov"
-#   SimpleCov.start do
-#     command_name "Job #{ENV["TEST_ENV_NUMBER"]}" if ENV["TEST_ENV_NUMBER"]
+if ENV["RUN_SIMPLECOV"]
+  require "simplecov"
+  SimpleCov.start do
+    command_name "Job #{ENV["TEST_ENV_NUMBER"]}" if ENV["TEST_ENV_NUMBER"]
 
-#     add_filter "/spec/"
-#     add_filter "/lib/tasks/auto_annotate_models.rake"
-#     add_group "Models", "/app/models"
-#     add_group "Controllers", "/app/controllers"
-#     add_group "Channels", "/app/channels"
-#     add_group "Decorators", "/app/decorators"
-#     add_group "Helpers", "/app/helpers"
-#     add_group "Jobs", "/app/jobs"
-#     add_group "Importers", "/app/lib/importers"
-#     add_group "Mailers", "/app/mailers"
-#     add_group "Policies", "/app/policies"
-#     add_group "Values", "/app/values"
-#     add_group "Tasks", "/lib/tasks"
-#     add_group "Config", "/config"
-#     add_group "Database", "/db"
-#   end
-#   # https://github.com/simplecov-ruby/simplecov?tab=readme-ov-file#want-to-use-spring-with-simplecov
-#   # Rails.application.eager_load!
-# end
+    add_filter "/spec/"
+    add_filter "/lib/tasks/auto_annotate_models.rake"
+    add_group "Models", "/app/models"
+    add_group "Controllers", "/app/controllers"
+    add_group "Channels", "/app/channels"
+    add_group "Decorators", "/app/decorators"
+    add_group "Helpers", "/app/helpers"
+    add_group "Jobs", "/app/jobs"
+    add_group "Importers", "/app/lib/importers"
+    add_group "Mailers", "/app/mailers"
+    add_group "Policies", "/app/policies"
+    add_group "Values", "/app/values"
+    add_group "Tasks", "/lib/tasks"
+    add_group "Config", "/config"
+    add_group "Database", "/db"
+  end
+  # https://github.com/simplecov-ruby/simplecov?tab=readme-ov-file#want-to-use-spring-with-simplecov
+  # Rails.application.eager_load!
+end
 
 # See https://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
 RSpec.configure do |config|

spec/support/rspec_retry.rb

@@ -0,0 +1,16 @@
+# spec/spec_helper.rb
+require "rspec/retry"
+
+RSpec.configure do |config|
+  # show retry status in spec process
+  config.verbose_retry = true
+  # show exception that triggers a retry if verbose_retry is set to true
+  config.display_try_failure_messages = true
+
+  if ENV["CI"] == "true"
+    # run retry only on features
+    config.around :each, :js do |ex|
+      ex.run_with_retry retry: 3
+    end
+  end
+end

spec/system/casa_cases/edit_spec.rb

@@ -44,17 +44,21 @@
 
       find(".ts-control").click
 
-      ts_checkboxes = page.all(".ts-dropdown-content input")
+      page.all(".ts-dropdown-content input")
 
       select_all_el = page.find("span[data-test=select-all-input]")
       # uncheck all contact type options
       select_all_el.click
-      ts_checkboxes.each do |el|
-        expect(el).not_to be_checked
+      within ".ts-dropdown-content" do
+        expect(page).not_to have_css(".form-check-input--checked")
+        expect(page).to have_css(".form-check-input--unchecked", count: 3)
       end
       # check all contact type options
       select_all_el.click
-      expect(ts_checkboxes).to all(be_checked)
+      within ".ts-dropdown-content" do
+        expect(page).not_to have_css("input.form-check-input--unchecked")
+        expect(page).to have_css("input.form-check-input--checked", count: 3)
+      end
 
       # unselect contact_type from dropdown
       find("span", text: contact_type.name).click
@@ -238,17 +242,20 @@
       expect(page).to have_text("Set Implementation Status")
 
       find(".ts-control").click
-      ts_checkboxes = page.all(".ts-dropdown-content input")
 
       select_all_el = page.find("span[data-test=select-all-input]")
       # uncheck all contact type options
       select_all_el.click
-      ts_checkboxes.each do |el|
-        expect(el).not_to be_checked
+      within ".ts-dropdown-content" do
+        expect(page).not_to have_css(".form-check-input--checked")
+        expect(page).to have_css(".form-check-input--unchecked", count: 2)
       end
       # check all contact type options
       select_all_el.click
-      expect(ts_checkboxes).to all(be_checked)
+      within ".ts-dropdown-content" do
+        expect(page).not_to have_css("input.form-check-input--unchecked")
+        expect(page).to have_css("input.form-check-input--checked", count: 2)
+      end
       # since all contact type options checked, don't need to select one
       within ".top-page-actions" do
         click_on "Update CASA Case"
@@ -317,6 +324,8 @@ def sign_in_and_assign_volunteer
       context "when a volunteer is assigned to a case" do
         it "marks the volunteer as assigned and shows the start date of the assignment", :js do
           sign_in_and_assign_volunteer
+          expect(page).to have_content("Volunteer assigned to case")
+
           expect(casa_case.case_assignments.count).to eq 1
 
           unassign_button = page.find("button.btn-outline-danger")

spec/system/casa_cases/new_spec.rb

@@ -34,16 +34,19 @@
 
           find(".ts-control").click
 
-          ts_checkboxes = page.all(".ts-dropdown-content input")
           select_all_el = page.find("span[data-test=select-all-input]")
           # uncheck all contact type options
           select_all_el.click
-          ts_checkboxes.each do |el|
-            expect(el).not_to be_checked
+          within ".ts-dropdown-content" do
+            expect(page).not_to have_css(".form-check-input--checked")
+            expect(page).to have_css(".form-check-input--unchecked", count: 2)
           end
           # check all contact type options
           select_all_el.click
-          expect(ts_checkboxes).to all(be_checked)
+          within ".ts-dropdown-content" do
+            expect(page).not_to have_css("input.form-check-input--unchecked")
+            expect(page).to have_css("input.form-check-input--checked", count: 2)
+          end
 
           select "Test User", from: "casa_case[case_assignments_attributes][0][volunteer_id]"
 

spec/system/case_contacts/contact_topic_answers_spec.rb

@@ -48,8 +48,9 @@ def notes_section
       {question: topic_two.question, answer: "Second discussion topic answer."}
     ])
 
-    expect { click_on "Submit" }
-      .to change(CaseContact.active, :count).by(1)
+    click_on "Submit"
+    expect(page).to have_content("Case contact successfully created.")
+    expect(CaseContact.active.size).to eq 1
 
     case_contact = CaseContact.active.last
     expect(case_contact.reload.contact_topic_answers).to be_present
@@ -64,7 +65,7 @@ def notes_section
     subject
     fill_in_contact_details(contact_types: [contact_type.name])
 
-    expect {
+    expect do
       using_wait_time 6 do # autosave debounce may be longer than capybara's wait time
         answer_topic contact_topics.first.question, "First discussion topic answer."
         within notes_section do
@@ -76,8 +77,8 @@ def notes_section
       end
 
       click_on "Submit"
-    }
-      .to change(CaseContact.active, :count).by(1)
+      expect(page).to have_content("Case contact successfully created.")
+    end.to change(CaseContact.active, :count).by(1)
       .and change(ContactTopicAnswer, :count).by(0) # answer already exists on page load
 
     case_contact = CaseContact.active.last
@@ -122,7 +123,10 @@ def notes_section
       fill_in_contact_details
       fill_in "Additional Notes", with: "This is a note."
 
-      expect { click_on "Submit" }.to change(CaseContact.active, :count).by(1)
+      expect do
+        click_on "Submit"
+        expect(page).to have_content("Case contact successfully created.")
+      end.to change(CaseContact.active, :count).by(1)
 
       contact = CaseContact.active.last
       expect(contact.contact_topic_answers).to be_empty
@@ -188,6 +192,7 @@ def notes_section
           expect(notes_section).to have_select(class: topic_select_class, count: 1, visible: :all)
 
           click_on "Submit"
+          expect(page).to have_content(/Case contact .* was successfully updated./)
         }
           .to change(ContactTopicAnswer, :count).by(-1)
 

spec/system/case_contacts/followups/resolve_spec.rb

@@ -11,11 +11,13 @@
   let(:case_contact) { build(:case_contact, casa_case: casa_case, creator: cc_creator) }
   let!(:followup) { create(:followup, case_contact: case_contact, creator: followup_creator) }
 
+  before { sign_in admin }
+
   it "changes status of followup to resolved" do
-    sign_in admin
     visit casa_case_path(case_contact.casa_case)
 
     click_button "Resolve Reminder"
+    expect(page).to have_button("Make Reminder")
 
     expect(case_contact.followups.count).to eq(1)
     expect(case_contact.followups.first.resolved?).to be_truthy
@@ -25,11 +27,13 @@
     let(:cc_creator) { volunteer }
     let(:followup_creator) { volunteer }
 
-    xit "changes status of followup to resolved" do # TODO make test not flaky
-      sign_in admin
+    before { sign_in admin }
+
+    it "changes status of followup to resolved" do # TODO make test not flaky
       visit casa_case_path(case_contact.casa_case)
 
       click_button "Resolve Reminder"
+      expect(page).to have_button("Make Reminder")
 
       expect(case_contact.followups.count).to eq(1)
       expect(case_contact.followups.first.resolved?).to be_truthy
@@ -40,6 +44,7 @@
       visit casa_case_path(case_contact.casa_case)
 
       click_button "Resolve Reminder"
+      expect(page).to have_button("Make Reminder")
 
       expect(page).to have_button("Make Reminder")
     end
@@ -49,11 +54,13 @@
     let(:cc_creator) { supervisor }
     let(:followup_creator) { volunteer }
 
+    before { sign_in supervisor }
+
     it "changes status of followup to resolved" do
-      sign_in supervisor
       visit casa_case_path(case_contact.casa_case)
 
       click_button "Resolve Reminder"
+      expect(page).to have_button("Make Reminder")
 
       expect(case_contact.followups.count).to eq(1)
       expect(case_contact.followups.first.resolved?).to be_truthy
@@ -66,13 +73,14 @@
 
     before do
       case_contact.casa_case.assigned_volunteers << volunteer
+      sign_in volunteer
     end
 
     it "changes status of followup to resolved" do
-      sign_in volunteer
       visit case_contacts_path
 
       click_button "Resolve Reminder"
+      expect(page).to have_button("Make Reminder")
 
       expect(case_contact.followups.count).to eq(1)
       expect(case_contact.followups.first.resolved?).to be_truthy