Fix benchmarks (#707)

* pull in JSON and HTML formatters, set them up correctly

* add numbers to preserve order

* remove a warning

* use Enum.each to avoid allocating data

Author: aseigo

bench/scripts/macro/all_bench.exs

@@ -35,13 +35,9 @@ jobs = %{
   "MyXQL Repo.all/2" => fn -> Ecto.Bench.MyXQLRepo.all(User, limit: limit) end
 }
 
-path = System.get_env("BENCHMARKS_OUTPUT_PATH") || "bench/results"
-file = Path.join(path, "all.json")
-
 Benchee.run(
   jobs,
-  formatters: [Benchee.Formatters.Console],
-  formatter_options: [json: [file: file]],
+  formatters: Ecto.Bench.Helper.formatters("all"),
   time: 10,
   after_each: fn results ->
     ^limit = length(results)

bench/scripts/macro/insert_bench.exs

@@ -29,14 +29,10 @@ jobs = %{
   "MyXQL Insert" => fn entry -> Ecto.Bench.MyXQLRepo.insert!(entry) end
 }
 
-path = System.get_env("BENCHMARKS_OUTPUT_PATH") || "bench/results"
-file = Path.join(path, "insert.json")
-
 Benchee.run(
   jobs,
   inputs: inputs,
-  formatters: [Benchee.Formatters.Console],
-  formatter_options: [json: [file: file]]
+  formatters: Ecto.Bench.Helper.formatters("insert")
 )
 
 # Clean inserted data

bench/scripts/micro/load_bench.exs

@@ -22,34 +22,34 @@ Code.require_file("../../support/setup.exs", __DIR__)
 alias Ecto.Bench.User
 
 inputs = %{
-  "Small 1 Thousand" =>
-    1..1_000 |> Enum.map(fn _ -> %{name: "Alice", email: "email@email.com"} end),
-  "Medium 100 Thousand" =>
-    1..100_000 |> Enum.map(fn _ -> %{name: "Alice", email: "email@email.com"} end),
-  "Big 1 Million" =>
-    1..1_000_000 |> Enum.map(fn _ -> %{name: "Alice", email: "email@email.com"} end),
-  "Time attr" =>
-    1..100_000 |> Enum.map(fn _ -> %{name: "Alice", time_attr: ~T[21:25:04.361140]} end),
-  "Date attr" => 1..100_000 |> Enum.map(fn _ -> %{name: "Alice", date_attr: ~D[2018-06-20]} end),
-  "NaiveDateTime attr" =>
+  "1. Small 1 Thousand" =>
+    1..1_000 |> Stream.map(fn _ -> %{name: "Alice", email: "email@email.com"} end),
+  "2. Medium 100 Thousand" =>
+    1..100_000 |> Stream.map(fn _ -> %{name: "Alice", email: "email@email.com"} end),
+  "3. Big 1 Million" =>
+    1..1_000_000 |> Stream.map(fn _ -> %{name: "Alice", email: "email@email.com"} end),
+  "4. Time attr" =>
+    1..100_000 |> Stream.map(fn _ -> %{name: "Alice", time_attr: ~T[21:25:04.361140]} end),
+  "5. Date attr" =>
+    1..100_000 |> Stream.map(fn _ -> %{name: "Alice", date_attr: ~D[2018-06-20]} end),
+  "6. NaiveDateTime attr" =>
     1..100_000
-    |> Enum.map(fn _ -> %{name: "Alice", naive_datetime_attr: ~N[2019-06-20 21:32:07.424178]} end),
-  "UUID attr" =>
+    |> Stream.map(fn _ ->
+      %{name: "Alice", naive_datetime_attr: ~N[2019-06-20 21:32:07.424178]}
+    end),
+  "7. UUID attr" =>
     1..100_000
-    |> Enum.map(fn _ -> %{name: "Alice", uuid: Ecto.UUID.bingenerate()} end)
+    |> Stream.map(fn _ -> %{name: "Alice", uuid: Ecto.UUID.bingenerate()} end)
 }
 
 jobs = %{
-  "Pg Loader" => fn data -> Enum.map(data, &Ecto.Bench.PgRepo.load(User, &1)) end,
-  "MyXQL Loader" => fn data -> Enum.map(data, &Ecto.Bench.MyXQLRepo.load(User, &1)) end
+  "Control" => fn stream -> Enum.each(stream, fn _data -> true end) end,
+  "Pg Loader" => fn stream -> Enum.each(stream, &Ecto.Bench.PgRepo.load(User, &1)) end,
+  "MyXQL Loader" => fn stream -> Enum.each(stream, &Ecto.Bench.MyXQLRepo.load(User, &1)) end
 }
 
-path = System.get_env("BENCHMARKS_OUTPUT_PATH") || "bench/results"
-file = Path.join(path, "load.json")
-
 Benchee.run(
   jobs,
   inputs: inputs,
-  formatters: [Benchee.Formatters.Console],
-  formatter_options: [json: [file: file]]
+  formatters: Ecto.Bench.Helper.formatters("load")
 )

bench/scripts/micro/to_sql_bench.exs

@@ -25,26 +25,26 @@ import Ecto.Query
 alias Ecto.Bench.{User, Game}
 
 inputs = %{
-  "Ordinary Select All" => {:all, from(User)},
-  "Ordinary Delete All" => {:delete_all, from(User)},
-  "Ordinary Update All" => {:update_all, from(User, update: [set: [name: "Thor"]])},
-  "Ordinary Where" => {:all, from(User, where: [name: "Thanos", email: "blah@blah"])},
-  "Fetch First Registry" => {:all, first(User)},
-  "Fetch Last Registry" => {:all, last(User)},
-  "Ordinary Order By" => {:all, order_by(User, desc: :name)},
-  "Complex Query 2 Joins" =>
+  "1. Ordinary Select All" => {:all, from(User)},
+  "2. Ordinary Delete All" => {:delete_all, from(User)},
+  "3. Ordinary Update All" => {:update_all, from(User, update: [set: [name: "Thor"]])},
+  "4. Ordinary Where" => {:all, from(User, where: [name: "Thanos", email: "blah@blah"])},
+  "5. Fetch First Registry" => {:all, first(User)},
+  "6. Fetch Last Registry" => {:all, last(User)},
+  "7. Ordinary Order By" => {:all, order_by(User, desc: :name)},
+  "8. Complex Query 2 Joins" =>
     {:all,
      from(User, where: [name: "Thanos"])
      |> join(:left, [u], ux in User, on: u.id == ux.id)
      |> join(:right, [j], uj in User, on: j.id == 1 and j.email == "email@email")
      |> select([u, ux], {u.name, ux.email})},
-  "Complex Query 4 Joins" =>
+  "9. Complex Query 4 Joins" =>
     {:all,
      from(User)
      |> join(:left, [u], g in Game, on: g.name == u.name)
      |> join(:right, [g], u in User, on: g.id == 1 and u.email == "email@email")
-     |> join(:inner, [u], g in fragment("SELECT * from games where game.id = ?", u.id))
-     |> join(:left, [g], u in fragment("SELECT * from users = ?", g.id))
+     |> join(:inner, [u], g in fragment("SELECT * from games where game.id = ?", u.id), on: true)
+     |> join(:left, [g], u in fragment("SELECT * from users = ?", g.id), on: true)
      |> select([u, g], {u.name, g.price})}
 }
 
@@ -53,12 +53,8 @@ jobs = %{
   "MyXQL Query Builder" => fn {type, query} -> Ecto.Bench.MyXQLRepo.to_sql(type, query) end
 }
 
-path = System.get_env("BENCHMARKS_OUTPUT_PATH") || "bench/results"
-file = Path.join(path, "to_sql.json")
-
 Benchee.run(
   jobs,
   inputs: inputs,
-  formatters: [Benchee.Formatters.Console],
-  formatter_options: [json: [file: file]]
+  formatters: Ecto.Bench.Helper.formatters("to_sql")
 )

bench/support/helper.exs

@@ -0,0 +1,13 @@
+defmodule Ecto.Bench.Helper do
+  def report_dir, do: System.get_env("BENCHMARKS_OUTPUT_PATH") || "bench/results"
+
+  def formatters(test_name) do
+    timestamp = DateTime.now!("Etc/UTC") |> DateTime.to_unix(:second)
+    file = Path.join(report_dir(), "#{timestamp}_#{test_name}")
+    [
+      Benchee.Formatters.Console,
+      {Benchee.Formatters.JSON, file: "#{file}.json"},
+      {Benchee.Formatters.HTML, file: "#{file}.html", auto_open: false}
+    ]
+  end
+end

bench/support/setup.exs

@@ -1,8 +1,12 @@
 Code.require_file("repo.exs", __DIR__)
 Code.require_file("migrations.exs", __DIR__)
 Code.require_file("schemas.exs", __DIR__)
+Code.require_file("helper.exs", __DIR__)
+
+File.mkdir_p(Ecto.Bench.Helper.report_dir())
 
 alias Ecto.Bench.{PgRepo, MyXQLRepo, CreateUser}
+alias Ecto.Bench.{PgRepo, CreateUser}
 
 {:ok, _} = Ecto.Adapters.Postgres.ensure_all_started(PgRepo.config(), :temporary)
 {:ok, _} = Ecto.Adapters.MyXQL.ensure_all_started(MyXQLRepo.config(), :temporary)

mix.exs

@@ -62,13 +62,15 @@ defmodule EctoSQL.MixProject do
       tds_dep(),
 
       # Bring something in for JSON during tests
-      {:jason, ">= 0.0.0", only: [:test, :docs]},
+      {:jason, "~> 1.0", only: [:test, :bench, :docs]},
 
       # Docs
       {:ex_doc, "~> 0.21", only: :docs},
 
       # Benchmarks
-      {:benchee, "~> 1.0", only: :bench}
+      {:benchee, "~> 1.0", only: :bench},
+      {:benchee_html, "~> 1.0", only: :bench},
+      {:benchee_json, "~> 1.0", only: :bench}
     ]
   end
 

mix.lock

@@ -1,5 +1,7 @@
 %{
   "benchee": {:hex, :benchee, "1.2.0", "afd2f0caec06ce3a70d9c91c514c0b58114636db9d83c2dc6bfd416656618353", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "ee729e53217898b8fd30aaad3cce61973dab61574ae6f48229fe7ff42d5e4457"},
+  "benchee_html": {:hex, :benchee_html, "1.0.1", "1e247c0886c3fdb0d3f4b184b653a8d6fb96e4ad0d0389267fe4f36968772e24", [:mix], [{:benchee, ">= 0.99.0 and < 2.0.0", [hex: :benchee, repo: "hexpm", optional: false]}, {:benchee_json, "~> 1.0", [hex: :benchee_json, repo: "hexpm", optional: false]}], "hexpm", "b00a181af7152431901e08f3fc9f7197ed43ff50421a8347b0c80bf45d5b3fef"},
+  "benchee_json": {:hex, :benchee_json, "1.0.0", "cc661f4454d5995c08fe10dd1f2f72f229c8f0fb1c96f6b327a8c8fc96a91fe5", [:mix], [{:benchee, ">= 0.99.0 and < 2.0.0", [hex: :benchee, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "da05d813f9123505f870344d68fb7c86a4f0f9074df7d7b7e2bb011a63ec231c"},
   "db_connection": {:hex, :db_connection, "2.8.0", "64fd82cfa6d8e25ec6660cea73e92a4cbc6a18b31343910427b702838c4b33b2", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "008399dae5eee1bf5caa6e86d204dcb44242c82b1ed5e22c881f2c34da201b15"},
   "decimal": {:hex, :decimal, "2.3.0", "3ad6255aa77b4a3c4f818171b12d237500e63525c2fd056699967a3e7ea20f62", [:mix], [], "hexpm", "a4d66355cb29cb47c3cf30e71329e58361cfcb37c34235ef3bf1d7bf3773aeac"},
   "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"},