Merge pull request #1206 from Crown-Commercial-Service/feature/nrmi-178-clean-up-non-active-val-failed-entries

Feature/nrmi 178 clean up non active val failed entries

Author: reemi1510

.rubocop.yml

@@ -122,6 +122,7 @@ Metrics/AbcSize:
         - 'app/controllers/admin/suppliers_controller.rb'
         - 'app/controllers/v1/tasks_controller.rb'
         - 'app/controllers/admin/users_controller.rb'
+        - 'app/jobs/cleanup_submission_entries_job.rb'
 
 Layout/LineLength:
     Max: 120

app/jobs/cleanup_submission_entries_job.rb

@@ -0,0 +1,58 @@
+class CleanupSubmissionEntriesJob < ApplicationJob
+  ENTRIES_BATCH_SIZE = 1000
+
+  # rubocop:disable Metrics/CyclomaticComplexity
+  # rubocop:disable Metrics/PerceivedComplexity
+  def perform(dry_run: false, max_run_time: 5.hours, task_batch_sixe: 100)
+    start_time = Time.current
+    total_deleted_entries = 0
+
+    tasks_with_unprocessed_submissions = Task.joins(:submissions)
+                                             .where(submissions: { aasm_state: 'validation_failed',
+cleanup_processed: false })
+                                             .distinct
+
+    tasks_with_unprocessed_submissions.find_in_batches(batch_size: task_batch_sixe) do |tasks_batch|
+      tasks_batch.each do |task|
+        active_id = task.active_submission&.id
+
+        failed_submissions = task.submissions
+                                 .where(aasm_state: 'validation_failed', cleanup_processed: false)
+                                 .where.not(id: active_id)
+
+        next if failed_submissions.empty?
+
+        deleted_for_task = 0
+
+        failed_submissions.find_each do |submission|
+          begin
+            submission.entries.in_batches(of: ENTRIES_BATCH_SIZE) do |entries_batch|
+              if dry_run
+                Rollbar.info("Dry run: would delete #{entries_batch.count} entries for Submission ID #{submission.id}.")
+              else
+                deleted_count = entries_batch.delete_all
+                deleted_for_task += deleted_count
+                total_deleted_entries += deleted_count
+                # rubocop:disable Layout/LineLength
+                Rollbar.info("Task ID #{task.id}: Processed #{failed_submissions.count} failed submissions, deleted #{deleted_for_task} entries.")
+                # rubocop:enable Layout/LineLength
+              end
+            end
+
+            submission.update(cleanup_processed: true) unless dry_run
+          rescue StandardError => e
+            Rollbar.error(e, "Error processing Submission ID #{submission.id} for Task ID #{task.id}: #{e.message}")
+          end
+
+          break if Time.current - start_time >= max_run_time
+        end
+      end
+
+      break if Time.current - start_time >= max_run_time
+
+      sleep 1 # To avoid overwhelming the database
+    end
+  end
+  # rubocop:enable Metrics/CyclomaticComplexity
+  # rubocop:enable Metrics/PerceivedComplexity
+end

config/application.rb

@@ -26,13 +26,14 @@ class Application < Rails::Application
 
     # config.active_record.yaml_column_permitted_classes = [Time]
 
-    console do
-      require './lib/console_helpers'
-      if defined?(Pry)
-        TOPLEVEL_BINDING.eval('self').extend ConsoleHelpers
-      else
-        Rails::ConsoleMethods.include ConsoleHelpers
-      end
-    end
+    # TODO: this is deprecated, figure out alternative
+    # console do
+    #   require './lib/console_helpers'
+    #   if defined?(Pry)
+    #     TOPLEVEL_BINDING.eval('self').extend ConsoleHelpers
+    #   else
+    #     Rails::ConsoleMethods.include ConsoleHelpers
+    #   end
+    # end
   end
 end

config/sidekiq_schedule.yml

@@ -10,3 +10,7 @@ kill_stuck_submissions:
   cron: '*/5 * * * *'
   class: KillStuckSubmissionsJob
   queue: default
+cleanup_submission_entries:
+  cron: '0 1 * * *'
+  class: CleanupSubmissionEntriesJob
+  queue: default

db/migrate/20251014121707_add_cleanup_processed_to_submissions.rb

@@ -0,0 +1,5 @@
+class AddCleanupProcessedToSubmissions < ActiveRecord::Migration[8.0]
+  def change
+    add_column :submissions, :cleanup_processed, :boolean, default: false, null: false
+  end
+end

db/migrate/20251020131628_add_index_to_submissions_on_task_id_and_aasm_state.rb

@@ -0,0 +1,5 @@
+class AddIndexToSubmissionsOnTaskIdAndAasmState < ActiveRecord::Migration[8.0]
+  def change
+    add_index :submissions, [:task_id, :aasm_state], name: 'index_submissions_on_task_id_and_aasm_state'
+  end
+end

db/migrate/20251020131706_add_index_to_submissions_cleanup_processed.rb

@@ -0,0 +1,5 @@
+class AddIndexToSubmissionsCleanupProcessed < ActiveRecord::Migration[8.0]
+  def change
+    add_index :submissions, :cleanup_processed
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[8.0].define(version: 2025_09_22_123744) do
+ActiveRecord::Schema[8.0].define(version: 2025_10_14_121707) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "citext"
   enable_extension "pg_catalog.plpgsql"
@@ -250,6 +250,7 @@
     t.datetime "submitted_at", precision: nil
     t.decimal "management_charge_total", precision: 18, scale: 4
     t.decimal "invoice_total", precision: 18, scale: 4
+    t.boolean "cleanup_processed", default: false, null: false
     t.index ["aasm_state"], name: "index_submissions_on_aasm_state"
     t.index ["created_at"], name: "index_submissions_on_created_at", order: :desc
     t.index ["created_by_id"], name: "index_submissions_on_created_by_id"

spec/jobs/cleanup_submission_entries_job_spec.rb

@@ -0,0 +1,48 @@
+require 'rails_helper'
+
+RSpec.describe CleanupSubmissionEntriesJob, type: :job do
+  describe '#perform' do
+    let!(:task) { FactoryBot.create(:task) }
+    let!(:failed_submission1) { FactoryBot.create(:submission_with_invalid_entries, task: task) }
+    let!(:failed_submission2) { FactoryBot.create(:submission_with_invalid_entries, task: task) }
+    let!(:submission_entry) { FactoryBot.create(:submission_entry, submission: failed_submission1) }
+
+    context 'when dry_run is true' do
+      it 'does not delete entries and logs the intended deletions' do
+        # rubocop:disable Layout/LineLength
+        expect(Rollbar).to receive(:info).with("Dry run: would delete 1 entries for Submission ID #{failed_submission1.id}.")
+        # rubocop:enable Layout/LineLength
+        expect do
+          described_class.perform_now(dry_run: true)
+        end.not_to change(SubmissionEntry, :count)
+        expect(failed_submission1.reload.cleanup_processed).to be_falsey
+        expect(failed_submission2.reload.cleanup_processed).to be_falsey
+      end
+    end
+
+    context 'when dry_run is false' do
+      it 'deletes entries and marks submissions as processed' do
+        expect(Rollbar).to receive(:info).with("Task ID #{task.id}: Processed 1 failed submissions, deleted 1 entries.")
+        expect do
+          described_class.perform_now(dry_run: false)
+        end.to change(SubmissionEntry, :count).by(-1)
+        expect(failed_submission1.reload.cleanup_processed).to be_truthy
+      end
+    end
+
+    context 'when there are no validation failed submissions' do
+      before do
+        failed_submission1.update(aasm_state: 'ingest_failed')
+        failed_submission2.update(aasm_state: 'completed')
+      end
+
+      it 'does not perform any deletions or updates' do
+        expect(Rollbar).not_to receive(:info)
+        expect do
+          described_class.perform_now(dry_run: false)
+        end.not_to change(SubmissionEntry, :count)
+        expect(failed_submission1.reload.cleanup_processed).to be_falsey
+      end
+    end
+  end
+end