pull in JSON and HTML formatters, set them up correctly

Author: aseigo

bench/scripts/macro/all_bench.exs

@@ -35,13 +35,9 @@ jobs = %{
   "MyXQL Repo.all/2" => fn -> Ecto.Bench.MyXQLRepo.all(User, limit: limit) end
 }
 
-path = System.get_env("BENCHMARKS_OUTPUT_PATH") || "bench/results"
-file = Path.join(path, "all.json")
-
 Benchee.run(
   jobs,
-  formatters: [Benchee.Formatters.Console],
-  formatter_options: [json: [file: file]],
+  formatters: Ecto.Bench.Helper.formatters("all"),
   time: 10,
   after_each: fn results ->
     ^limit = length(results)

bench/scripts/macro/insert_bench.exs

@@ -29,14 +29,10 @@ jobs = %{
   "MyXQL Insert" => fn entry -> Ecto.Bench.MyXQLRepo.insert!(entry) end
 }
 
-path = System.get_env("BENCHMARKS_OUTPUT_PATH") || "bench/results"
-file = Path.join(path, "insert.json")
-
 Benchee.run(
   jobs,
   inputs: inputs,
-  formatters: [Benchee.Formatters.Console],
-  formatter_options: [json: [file: file]]
+  formatters: Ecto.Bench.Helper.formatters("insert")
 )
 
 # Clean inserted data

bench/scripts/micro/load_bench.exs

@@ -44,12 +44,8 @@ jobs = %{
   "MyXQL Loader" => fn data -> Enum.map(data, &Ecto.Bench.MyXQLRepo.load(User, &1)) end
 }
 
-path = System.get_env("BENCHMARKS_OUTPUT_PATH") || "bench/results"
-file = Path.join(path, "load.json")
-
 Benchee.run(
   jobs,
   inputs: inputs,
-  formatters: [Benchee.Formatters.Console],
-  formatter_options: [json: [file: file]]
+  formatters: Ecto.Bench.Helper.formatters("load")
 )

bench/scripts/micro/to_sql_bench.exs

@@ -53,12 +53,8 @@ jobs = %{
   "MyXQL Query Builder" => fn {type, query} -> Ecto.Bench.MyXQLRepo.to_sql(type, query) end
 }
 
-path = System.get_env("BENCHMARKS_OUTPUT_PATH") || "bench/results"
-file = Path.join(path, "to_sql.json")
-
 Benchee.run(
   jobs,
   inputs: inputs,
-  formatters: [Benchee.Formatters.Console],
-  formatter_options: [json: [file: file]]
+  formatters: Ecto.Bench.Helper.formatters("to_sql")
 )

bench/support/helper.exs

@@ -0,0 +1,13 @@
+defmodule Ecto.Bench.Helper do
+  def report_dir, do: System.get_env("BENCHMARKS_OUTPUT_PATH") || "bench/results"
+
+  def formatters(test_name) do
+    timestamp = DateTime.now!("Etc/UTC") |> DateTime.to_unix(:second)
+    file = Path.join(report_dir(), "#{timestamp}_#{test_name}")
+    [
+      Benchee.Formatters.Console,
+      {Benchee.Formatters.JSON, file: "#{file}.json"},
+      {Benchee.Formatters.HTML, file: "#{file}.html", auto_open: false}
+    ]
+  end
+end

bench/support/setup.exs

@@ -1,8 +1,12 @@
 Code.require_file("repo.exs", __DIR__)
 Code.require_file("migrations.exs", __DIR__)
 Code.require_file("schemas.exs", __DIR__)
+Code.require_file("helper.exs", __DIR__)
+
+File.mkdir_p(Ecto.Bench.Helper.report_dir())
 
 alias Ecto.Bench.{PgRepo, MyXQLRepo, CreateUser}
+alias Ecto.Bench.{PgRepo, CreateUser}
 
 {:ok, _} = Ecto.Adapters.Postgres.ensure_all_started(PgRepo.config(), :temporary)
 {:ok, _} = Ecto.Adapters.MyXQL.ensure_all_started(MyXQLRepo.config(), :temporary)

mix.exs

@@ -62,13 +62,15 @@ defmodule EctoSQL.MixProject do
       tds_dep(),
 
       # Bring something in for JSON during tests
-      {:jason, ">= 0.0.0", only: [:test, :docs]},
+      {:jason, "~> 1.0", only: [:test, :bench, :docs]},
 
       # Docs
       {:ex_doc, "~> 0.21", only: :docs},
 
       # Benchmarks
-      {:benchee, "~> 1.0", only: :bench}
+      {:benchee, "~> 1.0", only: :bench},
+      {:benchee_html, "~> 1.0", only: :bench},
+      {:benchee_json, "~> 1.0", only: :bench}
     ]
   end
 

mix.lock

@@ -1,5 +1,7 @@
 %{
   "benchee": {:hex, :benchee, "1.2.0", "afd2f0caec06ce3a70d9c91c514c0b58114636db9d83c2dc6bfd416656618353", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "ee729e53217898b8fd30aaad3cce61973dab61574ae6f48229fe7ff42d5e4457"},
+  "benchee_html": {:hex, :benchee_html, "1.0.1", "1e247c0886c3fdb0d3f4b184b653a8d6fb96e4ad0d0389267fe4f36968772e24", [:mix], [{:benchee, ">= 0.99.0 and < 2.0.0", [hex: :benchee, repo: "hexpm", optional: false]}, {:benchee_json, "~> 1.0", [hex: :benchee_json, repo: "hexpm", optional: false]}], "hexpm", "b00a181af7152431901e08f3fc9f7197ed43ff50421a8347b0c80bf45d5b3fef"},
+  "benchee_json": {:hex, :benchee_json, "1.0.0", "cc661f4454d5995c08fe10dd1f2f72f229c8f0fb1c96f6b327a8c8fc96a91fe5", [:mix], [{:benchee, ">= 0.99.0 and < 2.0.0", [hex: :benchee, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "da05d813f9123505f870344d68fb7c86a4f0f9074df7d7b7e2bb011a63ec231c"},
   "db_connection": {:hex, :db_connection, "2.8.0", "64fd82cfa6d8e25ec6660cea73e92a4cbc6a18b31343910427b702838c4b33b2", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "008399dae5eee1bf5caa6e86d204dcb44242c82b1ed5e22c881f2c34da201b15"},
   "decimal": {:hex, :decimal, "2.3.0", "3ad6255aa77b4a3c4f818171b12d237500e63525c2fd056699967a3e7ea20f62", [:mix], [], "hexpm", "a4d66355cb29cb47c3cf30e71329e58361cfcb37c34235ef3bf1d7bf3773aeac"},
   "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"},