Split Benchee integration tests

Author: Will-thom

test/benchee/escript_test.exs

@@ -0,0 +1,20 @@
+defmodule Benchee.EscriptTest do
+  use ExUnit.Case, async: false
+
+  import Benchee.IntegrationHelpers
+
+  describe "escript building" do
+    @working_directory File.cwd!()
+    @sample_project_directory Path.expand("../fixtures/escript", __DIR__)
+    test "benchee can be built into and used as an escript" do
+      File.cd!(@sample_project_directory)
+      # we don't match the exit_status right now to get better error messages potentially
+      {output, exit_status} = System.cmd("bash", ["test.sh"])
+
+      readme_sample_asserts(output)
+      assert exit_status == 0
+    after
+      File.cd!(@working_directory)
+    end
+  end
+end

test/benchee/evaluation_warning_test.exs

@@ -0,0 +1,47 @@
+defmodule Benchee.EvaluationWarningTest do
+  use ExUnit.Case, async: true
+
+  describe "warn when functions are evaluated" do
+    test "warns when run in iex" do
+      # test env to avoid repeated compilation on CI
+      port = Port.open({:spawn, "iex -S mix"}, [:binary, env: [{~c"MIX_ENV", ~c"test"}]])
+
+      try do
+        # wait for startup
+        # timeout huge because of CI
+        assert_receive {^port, {:data, "iex(1)> "}}, 20_000
+
+        send(
+          port,
+          {self(),
+           {:command, "Benchee.run(%{\"test\" => fn -> 1 end}, time: 0.001, warmup: 0)\n"}}
+        )
+
+        assert_receive {^port, {:data, "Warning: " <> message}}, 20_000
+        assert message =~ ~r/test.+evaluated.+slower.+compiled.+module.+/is
+
+        # waiting for iex to be ready for input again/the benchmark to be finished
+        # sometimes we get "iex(2)>" as a separate message, sometimes it's attached to the
+        # previous output - hence we gotta doe out own `receive` checking.
+        assert :ok = wait_for_benchmark_finished(port)
+      after
+        # https://elixirforum.com/t/starting-shutting-down-iex-with-a-port-gracefully/60388/2?u=pragtob
+        send(port, {self(), {:command, "\a"}})
+        send(port, {self(), {:command, "q\n"}})
+      end
+    end
+
+    defp wait_for_benchmark_finished(port) do
+      receive do
+        {^port, {:data, output}} ->
+          if String.contains?(output, "iex(2)>") do
+            :ok
+          else
+            wait_for_benchmark_finished(port)
+          end
+      after
+        20_000 -> raise RuntimeError, "Waited too long for iex benchmark to finish/send iex(2)>"
+      end
+    end
+  end
+end

test/benchee/function_overhead_and_outliers_test.exs

@@ -0,0 +1,100 @@
+defmodule Benchee.FunctionOverheadAndOutliersTest do
+  use ExUnit.Case, async: false
+
+  import ExUnit.CaptureIO
+  import Benchee.IntegrationHelpers
+  import Benchee.TestHelpers
+  @test_configuration Benchee.IntegrationHelpers.test_configuration()
+
+  describe "function call overhead measurement" do
+    @overhead_output_regex ~r/function call overhead.*\d+/i
+    test "by default it is off" do
+      output =
+        capture_io(fn ->
+          Benchee.run(
+            %{"sleeps" => fn -> sleep_safe_time() end},
+            @test_configuration
+          )
+        end)
+
+      refute output =~ @overhead_output_regex
+    end
+
+    test "can be turned on" do
+      output =
+        capture_io(fn ->
+          Benchee.run(
+            %{"sleeps" => fn -> sleep_safe_time() end},
+            Keyword.merge(@test_configuration, measure_function_call_overhead: true)
+          )
+        end)
+
+      assert output =~ @overhead_output_regex
+    end
+  end
+
+  describe "exclude_outliers" do
+    test "even with it the high level README example still passes its asserts" do
+      output =
+        capture_io(fn ->
+          list = Enum.to_list(1..10_000)
+          map_fun = fn i -> [i, i * i] end
+
+          Benchee.run(
+            %{
+              "flat_map" => fn -> Enum.flat_map(list, map_fun) end,
+              "map.flatten" => fn -> list |> Enum.map(map_fun) |> List.flatten() end
+            },
+            Keyword.merge(@test_configuration, exclude_outliers: true)
+          )
+        end)
+
+      readme_sample_asserts(output)
+    end
+
+    # The easiest way to create an outlier is to just run something once
+    # and then take a "stable" measurement like reductions or memory
+    test "removes outliers" do
+      {:ok, agent} = Agent.start(fn -> 0 end)
+
+      output =
+        capture_io(fn ->
+          suite =
+            Benchee.run(
+              %{
+                "flawed" => fn ->
+                  # Produce some garbage but only once
+                  # can't use process dictionary as it's a different process every time
+                  if Agent.get(agent, & &1) < 1 do
+                    Enum.reduce(1..10_000, 0, &+/2)
+                    Agent.update(agent, &(&1 + 1))
+                  end
+                end
+              },
+              time: 0,
+              warmup: 0,
+              reduction_time: 0.05,
+              exclude_outliers: true
+            )
+
+          %{scenarios: [%{reductions_data: %{samples: samples, statistics: stats}}]} = suite
+
+          assert [outlier] = stats.outliers
+          assert outlier >= stats.upper_outlier_bound
+          # since the outlier is removed, all values are the same
+          assert stats.std_dev == 0
+          assert stats.minimum == stats.maximum
+
+          # It's a big outlier!
+          assert 10 * stats.average < outlier
+
+          # The outlier is only removed from the stats, but not from the samples
+          assert outlier in samples
+        end)
+
+      # As the outlier is removed, all measurements are the same
+      assert output =~ ~r/all.*reduction.*same/i
+      assert output =~ ~r/exclud.*outlier.*true/i
+    end
+  end
+end

test/benchee/hooks_test.exs

@@ -0,0 +1,65 @@
+defmodule Benchee.HooksTest do
+  use ExUnit.Case, async: true
+
+  import ExUnit.CaptureIO
+  import Benchee.TestHelpers
+  @test_configuration Benchee.IntegrationHelpers.test_configuration()
+
+  describe "hooks" do
+    test "it runs all of them" do
+      capture_io(fn ->
+        myself = self()
+
+        Benchee.run(
+          %{
+            "sleeper" =>
+              {fn -> sleep_safe_time() end,
+               before_each: fn input ->
+                 send(myself, :local_before)
+                 input
+               end,
+               after_each: fn _ -> send(myself, :local_after) end,
+               before_scenario: fn input ->
+                 send(myself, :local_before_scenario)
+                 input
+               end,
+               after_scenario: fn _ -> send(myself, :local_after_scenario) end},
+            "sleeper 2" => fn -> sleep_safe_time() end
+          },
+          Keyword.merge(
+            @test_configuration,
+            time: 0.0001,
+            warmup: 0,
+            before_each: fn input ->
+              send(myself, :global_before)
+              input
+            end,
+            after_each: fn _ -> send(myself, :global_after) end,
+            before_scenario: fn input ->
+              send(myself, :global_before_scenario)
+              input
+            end,
+            after_scenario: fn _ -> send(myself, :global_after_scenario) end
+          )
+        )
+      end)
+
+      assert_received_exactly([
+        # first job with all those local hooks
+        :global_before_scenario,
+        :local_before_scenario,
+        :global_before,
+        :local_before,
+        :local_after,
+        :global_after,
+        :local_after_scenario,
+        :global_after_scenario,
+        # second job that only runs global hooks
+        :global_before_scenario,
+        :global_before,
+        :global_after,
+        :global_after_scenario
+      ])
+    end
+  end
+end

test/benchee/max_sample_size_test.exs

@@ -0,0 +1,39 @@
+defmodule Benchee.MaxSampleSizeTest do
+  use ExUnit.Case, async: true
+
+  import ExUnit.CaptureIO
+  import Benchee.Configuration
+
+  alias Benchee.{Configuration, Suite}
+
+  @default_max_sample_size 1_000_000
+
+  test "max_sample_size defaults to 1 million samples" do
+    assert %Suite{configuration: %Configuration{max_sample_size: 1_000_000}} = init()
+  end
+
+  # Normally we do not test defaults this way, but this covers the full process
+  # using the default `max_sample_size` with formatters enabled.
+  @tag :performance
+  test "max_sample_size by default is set to 1 Million" do
+    capture_io(fn ->
+      suite =
+        Benchee.run(
+          %{
+            "fast" => fn -> :fast end
+          },
+          time: 1,
+          warmup: 0
+        )
+
+      Enum.each(suite.scenarios, fn scenario ->
+        assert scenario.run_time_data.statistics.sample_size <= @default_max_sample_size
+      end)
+    end)
+  end
+
+  test "max_sample_size can be disabled" do
+    assert %Suite{configuration: %Configuration{max_sample_size: nil}} =
+             init(max_sample_size: nil)
+  end
+end

test/benchee/measurement_integration_test.exs

@@ -0,0 +1,105 @@
+defmodule Benchee.MeasurementIntegrationTest do
+  use ExUnit.Case, async: true
+
+  import ExUnit.CaptureIO
+
+  @test_configuration Benchee.IntegrationHelpers.test_configuration()
+
+  describe "memory measurement" do
+    @describetag :memory_measure
+
+    test "measures memory usage when instructed to do so" do
+      output =
+        capture_io(fn ->
+          Benchee.run(
+            %{"To List" => fn -> Enum.to_list(1..100) end},
+            Keyword.merge(
+              @test_configuration,
+              memory_time: 0.001
+            )
+          )
+        end)
+
+      assert output =~ ~r/Memory usage statistics:/
+      assert output =~ ~r/To List\s+[0-9.]{3,} K*B{1}/
+    end
+
+    test "does not blow up when only measuring memory times" do
+      output =
+        capture_io(fn ->
+          Benchee.run(
+            %{
+              "something" => fn -> Enum.map(1..100, fn i -> i + 1 end) end
+            },
+            Keyword.merge(
+              @test_configuration,
+              time: 0,
+              warmup: 0,
+              memory_time: 0.001
+            )
+          )
+        end)
+
+      # no runtime statistics displayed
+      refute output =~ ~r/ips/i
+      assert output =~ ~r/memory.+statistics/i
+    end
+
+    test "the micro keyword list code from Michal does not break memory measurements #213" do
+      benches = %{
+        "delete old" => fn {kv, key} -> BenchKeyword.delete_v0(kv, key) end,
+        "delete reverse" => fn {kv, key} -> BenchKeyword.delete_v2(kv, key) end,
+        "delete keymember reverse" => fn {kv, key} -> BenchKeyword.delete_v3(kv, key) end,
+        "delete throw" => fn {kv, key} -> BenchKeyword.delete_v1(kv, key) end
+      }
+
+      inputs = %{
+        "large miss" => {Enum.map(1..100, &{:"k#{&1}", &1}), :k101},
+        "large hit" => {Enum.map(1..100, &{:"k#{&1}", &1}), :k100},
+        "small miss" => {Enum.map(1..10, &{:"k#{&1}", &1}), :k11}
+      }
+
+      output =
+        capture_io(fn ->
+          Benchee.run(
+            benches,
+            Keyword.merge(
+              @test_configuration,
+              inputs: inputs,
+              print: [fast_warning: false],
+              memory_time: 0.001,
+              warmup: 0,
+              time: 0
+            )
+          )
+        end)
+
+      refute output =~ "N/A"
+      refute output =~ ~r/warning/i
+      assert output =~ "large hit"
+      # Byte
+      assert output =~ "B"
+
+      assert output =~ ~r/1\.0\dx memory/
+
+      assert output =~ "∞ x memo"
+    end
+  end
+
+  describe "reduction measurement" do
+    test "measures reduction count when instructed to do so" do
+      output =
+        capture_io(fn ->
+          Benchee.run(
+            %{"To List" => fn -> Enum.to_list(1..100) end},
+            Keyword.merge(
+              @test_configuration,
+              reduction_time: 0.1
+            )
+          )
+        end)
+
+      assert output =~ ~r/Reduction count statistics:/
+    end
+  end
+end