Add --consolidated-results switch to log test results to JSON file

lib/parallel_tests/cli.rb

@@ -4,6 +4,7 @@
 require 'parallel_tests'
 require 'shellwords'
 require 'pathname'
+require 'json'
 
 module ParallelTests
   class CLI
@@ -70,6 +71,7 @@ def execute_in_parallel(items, num_processes, options)
 
     def run_tests_in_parallel(num_processes, options)
       test_results = nil
+      consolidated_results = {}
 
       run_tests_proc = -> do
         groups = @runner.tests_in_groups(options[:files], num_processes, options)
@@ -80,19 +82,28 @@ def run_tests_in_parallel(num_processes, options)
           num_processes = 1
         end
 
-        report_number_of_tests(groups) unless options[:quiet]
+        # Always populate consolidated results for tests info if consolidated results is requested
+        report_number_of_tests(groups, consolidated_results, options) if !options[:quiet] || options[:consolidated_results]
         test_results = execute_in_parallel(groups, groups.size, options) do |group, index|
           run_tests(group, index, num_processes, options)
         end
-        report_results(test_results, options) unless options[:quiet]
+        # Always populate consolidated results for summary if consolidated results is requested
+        report_results(test_results, consolidated_results, options) if !options[:quiet] || options[:consolidated_results]
       end
 
       if options[:quiet]
-        run_tests_proc.call
+        # Still capture timing if consolidated results are requested
+        if options[:consolidated_results]
+          report_time_taken(consolidated_results, options, &run_tests_proc)
+        else
+          run_tests_proc.call
+        end
       else
-        report_time_taken(&run_tests_proc)
+        report_time_taken(consolidated_results, options, &run_tests_proc)
       end
 
+      write_consolidated_results(consolidated_results, options)
+
       if any_test_failed?(test_results)
         warn final_fail_message
 
@@ -145,10 +156,14 @@ def lock(lockfile)
       end
     end
 
-    def report_results(test_results, options)
+    def report_results(test_results, consolidated_results, options)
       results = @runner.find_results(test_results.map { |result| result[:stdout] } * "")
-      puts ""
-      puts @runner.summarize_results(results)
+      @runner.summarize_results(results, consolidated_results)
+
+      unless options[:quiet]
+        puts ""
+        puts consolidated_results[:summary][:message]
+      end
 
       report_failure_rerun_commmand(test_results, options)
     end
@@ -167,12 +182,19 @@ def report_failure_rerun_commmand(test_results, options)
       end
     end
 
-    def report_number_of_tests(groups)
+    def report_number_of_tests(groups, consolidated_results, options = {})
       name = @runner.test_file_name
       num_processes = groups.size
       num_tests = groups.map(&:size).sum
       tests_per_process = (num_processes == 0 ? 0 : num_tests / num_processes)
-      puts "#{pluralize(num_processes, 'process')} for #{pluralize(num_tests, name)}, ~ #{pluralize(tests_per_process, name)} per process"
+      message = "#{pluralize(num_processes, 'process')} for #{pluralize(num_tests, name)}, ~ #{pluralize(tests_per_process, name)} per process"
+      consolidated_results[:tests] = {
+        message: message,
+        tests: num_tests,
+        processes: num_processes,
+        tests_per_process: tests_per_process
+      }
+      puts message unless options[:quiet]
     end
 
     def pluralize(n, singular)
@@ -185,6 +207,14 @@ def pluralize(n, singular)
       end
     end
 
+    def write_consolidated_results(consolidated_results, options)
+      return if options[:consolidated_results].nil? || options[:consolidated_results].empty?
+
+      File.write(options[:consolidated_results], consolidated_results.to_json)
+
+      puts "Wrote consolidated results to #{options[:consolidated_results]}"
+    end
+
     # exit with correct status code so rake parallel:test && echo 123 works
     def any_test_failed?(test_results)
       test_results.any? { |result| result[:exit_status] != 0 }
@@ -334,6 +364,7 @@ def parse_options!(argv)
         opts.on("--verbose-process-command", "Print the command that will be executed by each process before it begins") { options[:verbose_process_command] = true }
         opts.on("--verbose-rerun-command", "After a process fails, print the command executed by that process") { options[:verbose_rerun_command] = true }
         opts.on("--quiet", "Print only tests output") { options[:quiet] = true }
+        opts.on("--consolidated-results [PATH]", "Location to write consolidated test results") { |path| options[:consolidated_results] = path }
         opts.on("-v", "--version", "Show Version") do
           puts ParallelTests::VERSION
           exit 0
@@ -434,9 +465,14 @@ def execute_command_in_parallel(command, num_processes, options)
       abort if results.any? { |r| r[:exit_status] != 0 }
     end
 
-    def report_time_taken(&block)
+    def report_time_taken(consolidated_results, options = {}, &block)
       seconds = ParallelTests.delta(&block).to_i
-      puts "\nTook #{seconds} seconds#{detailed_duration(seconds)}"
+      message = "Took #{seconds} seconds#{detailed_duration(seconds)}"
+      consolidated_results[:time_taken] = {
+        seconds: seconds,
+        message: message
+      }
+      puts "\n#{message}" unless options[:quiet]
     end
 
     def detailed_duration(seconds)

lib/parallel_tests/cucumber/runner.rb

@@ -20,7 +20,7 @@ def line_is_result?(line)
           super || line =~ SCENARIO_REGEX || line =~ SCENARIOS_RESULTS_BOUNDARY_REGEX
         end
 
-        def summarize_results(results)
+        def summarize_results(results, consolidated_results)
           output = []
 
           scenario_groups = results.slice_before(SCENARIOS_RESULTS_BOUNDARY_REGEX).group_by(&:first)
@@ -31,7 +31,11 @@ def summarize_results(results)
 
           output << super
 
-          output.join("\n\n")
+          consolidated_results[:summary] = {
+            message: output.join("\n\n")
+          }
+
+          consolidated_results[:summary][:message]
         end
 
         def command_with_seed(cmd, seed)

lib/parallel_tests/gherkin/runner.rb

@@ -49,10 +49,10 @@ def build_test_command(file_list, options)
         # cucumber has 2 result lines per test run, that cannot be added
         # 1 scenario (1 failed)
         # 1 step (1 failed)
-        def summarize_results(results)
+        def summarize_results(results, consolidated_results)
           sort_order = ['scenario', 'step', 'failed', 'flaky', 'undefined', 'skipped', 'pending', 'passed']
 
-          ['scenario', 'step'].map do |group|
+          message = ['scenario', 'step'].map do |group|
             group_results = results.grep(/^\d+ #{group}/)
             next if group_results.empty?
 
@@ -64,6 +64,12 @@ def summarize_results(results)
             end
             "#{sums[0]} (#{sums[1..].join(", ")})"
           end.compact.join("\n")
+
+          consolidated_results[:summary] = {
+            message: message
+          }
+
+          consolidated_results[:summary][:message]
         end
 
         def cucumber_opts(given)

lib/parallel_tests/rspec/runner.rb

@@ -57,7 +57,7 @@ def command_with_seed(cmd, seed)
 
         # Summarize results from threads and colorize results based on failure and pending counts.
         #
-        def summarize_results(results)
+        def summarize_results(results, consolidated_results)
           text = super
           return text unless $stdout.tty?
           sums = sum_up_results(results)
@@ -69,7 +69,12 @@ def summarize_results(results)
             else
               32 # green
             end
-          "\e[#{color}m#{text}\e[0m"
+          colorized_text = "\e[#{color}m#{text}\e[0m"
+
+          # Update consolidated results with colorized message
+          consolidated_results[:summary][:message] = colorized_text
+
+          colorized_text
         end
 
         private

lib/parallel_tests/test/runner.rb

@@ -135,9 +135,15 @@ def test_env_number(process_number, options = {})
           end
         end
 
-        def summarize_results(results)
+        def summarize_results(results, consolidated_results)
           sums = sum_up_results(results)
-          sums.sort.map { |word, number| "#{number} #{word}#{'s' if number != 1}" }.join(', ')
+          message = sums.sort.map { |word, number| "#{number} #{word}#{'s' if number != 1}" }.join(', ')
+
+          consolidated_results[:summary] = sums.merge({
+            message: message
+          })
+
+          consolidated_results[:summary][:message]
         end
 
         # remove old seed and add new seed

spec/parallel_tests/cucumber/runner_spec.rb

@@ -21,7 +21,7 @@ def call(*args)
           "Failing Scenarios:", "cucumber features/failure:3", "cucumber features/failure:4",
           "Failing Scenarios:", "cucumber features/failure:5", "cucumber features/failure:6"
         ]
-        output = call(results)
+        output = call(results, {})
         output.gsub!(/.*WARNING.*\n/, "")
         expect(output).to eq(<<~TXT)
           Failing Scenarios:
@@ -42,7 +42,7 @@ def call(*args)
           "Failing Scenarios:", "cucumber features/failure:5", "cucumber features/failure:6",
           "Flaky Scenarios:", "cucumber features/failure:7", "cucumber features/failure:8"
         ]
-        expect(call(results)).to eq("Failing Scenarios:\ncucumber features/failure:1\ncucumber features/failure:2\ncucumber features/failure:5\ncucumber features/failure:6\n\nFlaky Scenarios:\ncucumber features/failure:3\ncucumber features/failure:4\ncucumber features/failure:7\ncucumber features/failure:8\n\n")
+        expect(call(results, {})).to eq("Failing Scenarios:\ncucumber features/failure:1\ncucumber features/failure:2\ncucumber features/failure:5\ncucumber features/failure:6\n\nFlaky Scenarios:\ncucumber features/failure:3\ncucumber features/failure:4\ncucumber features/failure:7\ncucumber features/failure:8\n\n")
       end
     end
   end

spec/parallel_tests/gherkin/runner_behaviour.rb

@@ -169,27 +169,27 @@ def call(*args)
         "4 scenarios (4 passed)", "40 steps (40 passed)",
         "1 scenario (1 passed)", "1 step (1 passed)"
       ]
-      expect(call(results)).to eq("12 scenarios (2 failed, 1 flaky, 9 passed)\n74 steps (3 failed, 2 skipped, 69 passed)")
+      expect(call(results, {})).to eq("12 scenarios (2 failed, 1 flaky, 9 passed)\n74 steps (3 failed, 2 skipped, 69 passed)")
     end
 
     it "adds same results with plurals" do
       results = [
         "1 scenario (1 passed)", "2 steps (2 passed)",
         "2 scenarios (2 passed)", "7 steps (7 passed)"
       ]
-      expect(call(results)).to eq("3 scenarios (3 passed)\n9 steps (9 passed)")
+      expect(call(results, {})).to eq("3 scenarios (3 passed)\n9 steps (9 passed)")
     end
 
     it "adds non-similar results" do
       results = [
         "1 scenario (1 passed)", "1 step (1 passed)",
         "2 scenarios (1 failed, 1 pending)", "2 steps (1 failed, 1 pending)"
       ]
-      expect(call(results)).to eq("3 scenarios (1 failed, 1 pending, 1 passed)\n3 steps (1 failed, 1 pending, 1 passed)")
+      expect(call(results, {})).to eq("3 scenarios (1 failed, 1 pending, 1 passed)\n3 steps (1 failed, 1 pending, 1 passed)")
     end
 
     it "does not pluralize 1" do
-      expect(call(["1 scenario (1 passed)", "1 step (1 passed)"])).to eq("1 scenario (1 passed)\n1 step (1 passed)")
+      expect(call(["1 scenario (1 passed)", "1 step (1 passed)"], {})).to eq("1 scenario (1 passed)\n1 step (1 passed)")
     end
   end
 

spec/parallel_tests/rspec/runner_spec.rb

@@ -137,7 +137,7 @@ def call(*args)
       before { allow($stdout).to receive(:tty?).and_return false }
 
       it 'is not colourized' do
-        results = ParallelTests::RSpec::Runner.send(:summarize_results, ['1 example, 0 failures, 0 pendings'])
+        results = ParallelTests::RSpec::Runner.send(:summarize_results, ['1 example, 0 failures, 0 pendings'], {})
 
         expect(results).to eq('1 example, 0 failures, 0 pendings')
       end
@@ -146,7 +146,7 @@ def call(*args)
     context 'on TTY device' do
       before { allow($stdout).to receive(:tty?).and_return true }
 
-      subject(:colorized_results) { ParallelTests::RSpec::Runner.send(:summarize_results, [result_string]) }
+      subject(:colorized_results) { ParallelTests::RSpec::Runner.send(:summarize_results, [result_string], {}) }
 
       context 'when there are no pending or failed tests' do
         let(:result_string) { '1 example, 0 failures, 0 pendings' }

spec/parallel_tests/test/runner_spec.rb

@@ -404,23 +404,23 @@ def call(*args)
     end
 
     it "adds results" do
-      expect(call(['1 foo 3 bar', '2 foo 5 bar'])).to eq('8 bars, 3 foos')
+      expect(call(['1 foo 3 bar', '2 foo 5 bar'], {})).to eq('8 bars, 3 foos')
     end
 
     it "adds results with braces" do
-      expect(call(['1 foo(s) 3 bar(s)', '2 foo 5 bar'])).to eq('8 bars, 3 foos')
+      expect(call(['1 foo(s) 3 bar(s)', '2 foo 5 bar'], {})).to eq('8 bars, 3 foos')
     end
 
     it "adds same results with plurals" do
-      expect(call(['1 foo 3 bar', '2 foos 5 bar'])).to eq('8 bars, 3 foos')
+      expect(call(['1 foo 3 bar', '2 foos 5 bar'], {})).to eq('8 bars, 3 foos')
     end
 
     it "adds non-similar results" do
-      expect(call(['1 xxx 2 yyy', '1 xxx 2 zzz'])).to eq('2 xxxs, 2 yyys, 2 zzzs')
+      expect(call(['1 xxx 2 yyy', '1 xxx 2 zzz'], {})).to eq('2 xxxs, 2 yyys, 2 zzzs')
     end
 
     it "does not pluralize 1" do
-      expect(call(['1 xxx 2 yyy'])).to eq('1 xxx, 2 yyys')
+      expect(call(['1 xxx 2 yyy'], {})).to eq('1 xxx, 2 yyys')
     end
   end