Dp 2398 tind spread validate header (#43)

* validating header for tind spreadsheet validator

* rubocop %w instead of quotes in array

* Added some comments for header validation description for tind spread validator

* Changed regular expression for tind spreadsheet header validation

Author: davezuckerman

app/lib/tind_spread/tind_batch.rb

@@ -27,15 +27,18 @@ def format_errors
     end
 
     def attachments(attachment_name)
-      if @all_errors.empty?
-        { "#{attachment_name}.csv" => @csv.to_s }
-      elsif @csv.count("\n") > 1
-        { "#{attachment_name}.csv" => @csv.to_s, "ERRORREPORT_#{attachment_name}.csv" => @errors_csv.to_s,
-          "ERRORREPORT_#{attachment_name}.txt" => format_errors }
-      else
-        { "ERRORREPORT_#{attachment_name}.csv" => @errors_csv.to_s,
-          "ERRORREPORT_#{attachment_name}.txt" => format_errors }
+      return { "ERRORREPORT_#{attachment_name}.txt" => format_errors } if @all_errors.key?(1)
+
+      return { "#{attachment_name}.csv" => @csv.to_s } if @all_errors.empty?
+
+      if @csv.count("\n") > 1
+        return { "#{attachment_name}.csv" => @csv.to_s,
+                 "ERRORREPORT_#{attachment_name}.csv" => @errors_csv.to_s,
+                 "ERRORREPORT_#{attachment_name}.txt" => format_errors }
       end
+
+      { "ERRORREPORT_#{attachment_name}.csv" => @errors_csv.to_s,
+        "ERRORREPORT_#{attachment_name}.txt" => format_errors }
     end
 
     # rubocop:disable Metrics/AbcSize
@@ -65,19 +68,37 @@ def create_rows(all_rows)
     end
     # rubocop:enable Metrics/MethodLength
 
+    def validate_header_row(headers)
+      header_errors = []
+      headers.each do |header|
+        header_errors << "Invalid header name: #{header.gsub(/^\d+:/, '')}" unless TindSpread::TindValidation.valid_header?(header)
+      end
+      header_errors
+    end
+
+    # rubocop:disable Metrics/MethodLength
     def run
       t = TindSpread::SpreadTool.new(@xlsx_path, @extension, @form_info[:directory])
       all_rows = t.spread
+      @all_errors = {}
+
+      # Validate header row
+      header_errors = validate_header_row(all_rows.first.keys)
+      @all_errors[1] = header_errors if header_errors.any?
+
+      return send_email if header_errors.any?
+
       @csv = TindSpread::MakeBatch.make_header(t.header(all_rows.first.keys), @form_info).encode('UTF-8')
       @errors_csv = TindSpread::MakeBatch.make_header(t.header(all_rows.first.keys), @form_info, remove_filename: false).encode('UTF-8')
-      @all_errors = {}
+
       create_rows(all_rows)
       @csv.to_s.gsub!("\xEF\xBB\xBF".force_encoding('UTF-8'), '')
       @errors_csv.to_s.gsub!("\xEF\xBB\xBF".force_encoding('UTF-8'), '')
       # File.write('output.csv', @csv)
       # File.write('errors.csv', @errors_csv)
       send_email
     end
+    # rubocop:enable Metrics/MethodLength
     # rubocop:enable Metrics/AbcSize
   end
 end

app/lib/tind_spread/tind_validation.rb

@@ -1,8 +1,17 @@
 require 'net/http'
 require 'open-uri'
 module TindSpread
+  # rubocop:disable Metrics/ModuleLength
   module TindValidation
 
+    # validates the header row
+    # should be 3 digits for field, 2 for indicator (can be _, number), one digit or number for subfield
+    # optionally can have a ('-' followed by a number). This is used to group columns into similar fields
+    # the header row can also be just "Filename" or "FFT". The program will create the proper fields for those
+    def self.valid_header?(str)
+      str.match?(/\d{3}[_|\d]{2}[a-zA-Z0-9](-\d+)?$/) || str.match?(/Filename|FFT/i)
+    end
+
     # runs a set of validations against a single row.
     # Row should be an array of hashes, key being the column header for the row.
     # rubocop:disable Metrics/MethodLength
@@ -28,6 +37,7 @@ def self.validate_row(row)
 
     # private
     class << self
+
       private
 
       def filename_error(row, errors)
@@ -149,4 +159,5 @@ def valid_500__3?(key, row)
 
     end
   end
+  # rubocop:enable Metrics/ModuleLength
 end

spec/lib/tind_spread/tind_batch_spec.rb

@@ -15,13 +15,13 @@
   let(:args) { { directory:, '982__a': 'test' } }
   let(:tind_batch) { described_class.new(args, xlsx, extension, email) }
   let(:spread_tool) { instance_double(TindSpread::SpreadTool) }
-  let(:all_rows) { [{ 'Header1' => 'Data1', 'Header2' => 'Data2' }, { 'Header1' => 'Data3', 'Header2' => 'Data4' }] }
+  let(:all_rows) { [{ '001__a' => 'Data1', '245__a' => 'Data2' }, { '001__a' => 'Data3', '245__a' => 'Data4' }] }
 
   before do
     allow(TindSpread::SpreadTool).to receive(:new).with(xlsx, extension, directory).and_return(spread_tool)
     allow(spread_tool).to receive(:spread).and_return(all_rows)
-    allow(spread_tool).to receive(:header).with(any_args).and_return(%w[Header1 Header2])
-    allow(TindSpread::MakeBatch).to receive(:make_header).with(any_args).and_return("Header1,Header2\n")
+    allow(spread_tool).to receive(:header).with(any_args).and_return(%w[001__a 245__a])
+    allow(TindSpread::MakeBatch).to receive(:make_header).with(any_args).and_return("001__a,245__a\n")
     allow(TindSpread::MakeBatch).to receive(:add_row).with(any_args).and_return("Data1,Data2\n")
     allow(TindSpread::TindValidation).to receive(:validate_row).with(any_args).and_return([])
     # rubocop:disable RSpec/MessageChain
@@ -49,15 +49,15 @@
   describe '#send_email' do
     it 'sends an email with the correct attachments' do
       tind_batch.instance_variable_set(:@all_errors, {})
-      tind_batch.instance_variable_set(:@csv, "Header1,Header2\nData1,Data2\n")
-      tind_batch.instance_variable_set(:@errors_csv, "Header1,Header2\n")
+      tind_batch.instance_variable_set(:@csv, "001__a,245__a\nData1,Data2\n")
+      tind_batch.instance_variable_set(:@errors_csv, "001__a,245__a\n")
       allow(Time).to receive(:current).and_return(Time.parse('2023-10-01 12:00:00 UTC'))
       attachment_name = 'test_2023-10-01'
       expect(RequestMailer).to receive(:tind_spread_email).with(
         email,
         'Tind batch load for test',
         'No errors found',
-        { "#{attachment_name}.csv" => "Header1,Header2\nData1,Data2\n" }
+        { "#{attachment_name}.csv" => "001__a,245__a\nData1,Data2\n" }
       ).and_return(double(deliver_now: true))
       tind_batch.send_email
     end
@@ -74,12 +74,34 @@
     end
   end
 
+  describe '#validate_header_row' do
+    it 'returns an empty array for valid headers' do
+      headers = %w[001__a 245__a 500__3]
+      errors = tind_batch.validate_header_row(headers)
+      expect(errors).to be_empty
+    end
+
+    it 'returns error messages for invalid headers' do
+      headers = %w[Header1 Header2]
+      errors = tind_batch.validate_header_row(headers)
+      expect(errors).to include('Invalid header name: Header1')
+      expect(errors).to include('Invalid header name: Header2')
+    end
+
+    it 'returns errors only for invalid headers in a mixed list' do
+      headers = %w[001__a InvalidHeader 245__a]
+      errors = tind_batch.validate_header_row(headers)
+      expect(errors).to include('Invalid header name: InvalidHeader')
+      expect(errors.length).to eq(1)
+    end
+  end
+
   describe '#run' do
     it 'runs the batch process' do
       allow(tind_batch).to receive(:send_email)
       tind_batch.run
-      expect(tind_batch.instance_variable_get(:@csv)).to eq("Header1,Header2\nData1,Data2\nData1,Data2\n")
-      expect(tind_batch.instance_variable_get(:@errors_csv)).to eq("Header1,Header2\n")
+      expect(tind_batch.instance_variable_get(:@csv)).to eq("001__a,245__a\nData1,Data2\nData1,Data2\n")
+      expect(tind_batch.instance_variable_get(:@errors_csv)).to eq("001__a,245__a\n")
     end
   end
 end

spec/lib/tind_spread/tind_validation_spec.rb

@@ -108,4 +108,26 @@
       expect(described_class.send(:corresponding_6?, '800__6', row)).to be false
     end
   end
+
+  describe '.valid_header?' do
+    it 'returns true for a valid header with standard format' do
+      expect(described_class.valid_header?('001__a')).to be true
+      expect(described_class.valid_header?('245__a')).to be true
+      expect(described_class.valid_header?('Filename')).to be true
+      expect(described_class.valid_header?('100_1a')).to be true
+    end
+
+    it 'returns true for a valid header with suffix format' do
+      expect(described_class.valid_header?('001__a-1')).to be true
+      expect(described_class.valid_header?('245__a-2')).to be true
+      expect(described_class.valid_header?('500__3-5')).to be true
+    end
+
+    it 'returns false for invalid headers' do
+      expect(described_class.valid_header?('abc')).to be false
+      expect(described_class.valid_header?('12__a')).to be false
+      expect(described_class.valid_header?('001__a-')).to be false
+      expect(described_class.valid_header?('001__a-ab')).to be false
+    end
+  end
 end