Put integration tests into a new integration folder and explain what it is there for

Author: PragTob

test/benchee/integration/README.md

@@ -0,0 +1,3 @@
+These tests are split out of `benchee_test.exs` to be able to run more of them in parallel.
+
+These are integration-y tests executing the whole stack.

test/benchee/escript_test.exs test/benchee/integration/escript_test.exstest/benchee/escript_test.exs renamed to test/benchee/integration/escript_test.exs

@@ -1,11 +1,12 @@
 defmodule Benchee.EscriptTest do
+  # Cding is global, so can't run in parallel
   use ExUnit.Case, async: false
 
   import Benchee.IntegrationHelpers
 
   describe "escript building" do
     @working_directory File.cwd!()
-    @sample_project_directory Path.expand("../fixtures/escript", __DIR__)
+    @sample_project_directory Path.expand("../../fixtures/escript", __DIR__)
     test "benchee can be built into and used as an escript" do
       File.cd!(@sample_project_directory)
       # we don't match the exit_status right now to get better error messages potentially

test/benchee/evaluation_warning_test.exs …/integration/evaluation_warning_test.exstest/benchee/evaluation_warning_test.exs renamed to test/benchee/integration/evaluation_warning_test.exs test/benchee/evaluation_warning_test.exs renamed to test/benchee/integration/evaluation_warning_test.exs

@@ -1,5 +1,5 @@
 defmodule Benchee.FunctionOverheadAndOutliersTest do
-  use ExUnit.Case, async: false
+  use ExUnit.Case, async: true
 
   import ExUnit.CaptureIO
   import Benchee.IntegrationHelpers

…/function_overhead_and_outliers_test.exs …/function_overhead_and_outliers_test.exstest/benchee/function_overhead_and_outliers_test.exs renamed to test/benchee/integration/function_overhead_and_outliers_test.exs test/benchee/function_overhead_and_outliers_test.exs renamed to test/benchee/integration/function_overhead_and_outliers_test.exs

@@ -14,6 +14,8 @@ defmodule BencheeTest do
   import Benchee.IntegrationHelpers
   import Benchee.TestHelpers
 
+  # Several integration tests are split out of this file into their own files so more tests can run in parallel
+
   @header_regex ~r/^Name.+ips.+average.+deviation.+median.+99th %$/m
   @test_configuration [time: 0.01, warmup: 0.005]