Only use async: true for ConnCase error tests on Postgres and no-Ecto apps (#6783)

Fixes a long-standing flaky test issue in generated SQLite applications where
running `mix test` would intermittently fail with:

    ** (Exqlite.Error) Database busy

Root Cause

`phx.new` generated `error_html_test.exs` and `error_json_test.exs` with
hardcoded `use ConnCase, async: true`.

In apps with Ecto enabled, `ConnCase` includes a setup block that calls
`DataCase.setup_sandbox(tags)`. When ExUnit runs an `async: true` test module
concurrently with `async: false` test modules (such as tests generated by
`phx.gen.auth`), `setup_sandbox` sees `tags[:async] == true` and checks out
a non-shared secondary connection handle (`shared: false`) from the Repo pool.

Because SQLite uses single-file database locking, having two active database
connections opening sandbox transactions concurrently causes file lock contention
whenever a synchronous test inserts records (e.g. `user_fixture()`). Depending
on ExUnit test seeding and process scheduling, the secondary connection checkout
fails with `(Exqlite.Error) Database busy`.

Background

When `--database sqlite3` support was added in 5143696
(#4268, April 2021), error controller test templates retained hardcoded
`use ConnCase, async: true`. Later, when `phx.gen.auth` introduced support for
`async: true` in 3d609e0 (#5689, January 2024),
it added a helper to conditionally append `async: true` ONLY for PostgreSQL adapters
while leaving non-Postgres adapters as `async: false`. However, `phx.new` was not
updated at the time to use the same logic for its generated error test templates.

Fix

1. Add `test_case_options/1` in `Phx.New.Generator` matching the behavior of `phx.gen.auth`:
   - `Ecto.Adapters.Postgres` -> `", async: true"`
   - `nil` (no Ecto) -> `", async: true"`
   - non-Postgres adapters (SQLite, MySQL, TDS) -> `""`

2. Update `error_html_test.exs.eex` and `error_json_test.exs.eex` templates to use `<%= @test_case_options %>`.

3. Added test coverage across single app and umbrella project generators for Postgres, `--no-ecto`, SQLite3, MySQL, and MSSQL.

Author: rhcarvalho

installer/lib/phx_new/generator.ex

@@ -326,7 +326,8 @@ defmodule Phx.New.Generator do
       elixir_install_bin_path: from_elixir_install && elixir_install_bin_path(),
       inside_docker_env?: inside_docker_env?,
       agents_md: agents_md,
-      config_regex_E: Version.match?(System.version(), "~> 1.19.3 or ~> 1.20") && "E" || ""
+      config_regex_E: Version.match?(System.version(), "~> 1.19.3 or ~> 1.20") && "E" || "",
+      test_case_options: test_case_options(adapter_module)
     ]
 
     %{project | binding: binding}
@@ -410,6 +411,10 @@ defmodule Phx.New.Generator do
     Mix.raise("Unknown database #{inspect(db)}")
   end
 
+  defp test_case_options(Ecto.Adapters.Postgres), do: ", async: true"
+  defp test_case_options(nil), do: ", async: true"
+  defp test_case_options(adapter) when is_atom(adapter), do: ""
+
   defp get_web_adapter("cowboy"),
     do:
       {:plug_cowboy, "~> 2.7", Phoenix.Endpoint.Cowboy2Adapter,

installer/templates/phx_test/controllers/error_html_test.exs.eex

@@ -1,5 +1,5 @@
 defmodule <%= @web_namespace %>.ErrorHTMLTest do
-  use <%= @web_namespace %>.ConnCase, async: true
+  use <%= @web_namespace %>.ConnCase<%= @test_case_options %>
 
   # Bring render_to_string/4 for testing custom views
   import Phoenix.Template, only: [render_to_string: 4]

installer/templates/phx_test/controllers/error_json_test.exs.eex

@@ -1,5 +1,5 @@
 defmodule <%= @web_namespace %>.ErrorJSONTest do
-  use <%= @web_namespace %>.ConnCase, async: true
+  use <%= @web_namespace %>.ConnCase<%= @test_case_options %>
 
   test "renders 404" do
     assert <%= @web_namespace %>.ErrorJSON.render("404.json", %{}) == %{errors: %{detail: "Not Found"}}

installer/test/phx_new_test.exs

@@ -89,8 +89,8 @@ defmodule Mix.Tasks.Phx.NewTest do
       end)
 
       assert_file("phx_blog/test/phx_blog_web/controllers/page_controller_test.exs")
-      assert_file("phx_blog/test/phx_blog_web/controllers/error_html_test.exs")
-      assert_file("phx_blog/test/phx_blog_web/controllers/error_json_test.exs")
+      assert_file("phx_blog/test/phx_blog_web/controllers/error_html_test.exs", "async: true")
+      assert_file("phx_blog/test/phx_blog_web/controllers/error_json_test.exs", "async: true")
       assert_file("phx_blog/test/support/conn_case.ex")
       assert_file("phx_blog/test/test_helper.exs")
 
@@ -557,6 +557,9 @@ defmodule Mix.Tasks.Phx.NewTest do
         assert file =~ "inputs: [\"*.{heex,ex,exs}\", \"{config,lib,test}/**/*.{heex,ex,exs}\"]"
         refute file =~ "subdirectories:"
       end)
+
+      assert_file("phx_blog/test/phx_blog_web/controllers/error_html_test.exs", "async: true")
+      assert_file("phx_blog/test/phx_blog_web/controllers/error_json_test.exs", "async: true")
     end)
   end
 
@@ -686,6 +689,14 @@ defmodule Mix.Tasks.Phx.NewTest do
         "custom_path/test/support/data_case.ex",
         "Ecto.Adapters.SQL.Sandbox.start_owner"
       )
+
+      assert_file("custom_path/test/custom_path_web/controllers/error_html_test.exs", fn file ->
+        refute file =~ "async: true"
+      end)
+
+      assert_file("custom_path/test/custom_path_web/controllers/error_json_test.exs", fn file ->
+        refute file =~ "async: true"
+      end)
     end)
   end
 
@@ -716,8 +727,18 @@ defmodule Mix.Tasks.Phx.NewTest do
         "Ecto.Adapters.SQL.Sandbox.start_owner"
       )
 
-      assert_file("custom_path/.gitignore", "*.db")
-      assert_file("custom_path/.gitignore", "*.db-*")
+      assert_file("custom_path/test/custom_path_web/controllers/error_html_test.exs", fn file ->
+        refute file =~ "async: true"
+      end)
+
+      assert_file("custom_path/test/custom_path_web/controllers/error_json_test.exs", fn file ->
+        refute file =~ "async: true"
+      end)
+
+      assert_file("custom_path/.gitignore", fn file ->
+        assert file =~ "*.db"
+        assert file =~ "*.db-*"
+      end)
     end)
   end
 
@@ -747,6 +768,14 @@ defmodule Mix.Tasks.Phx.NewTest do
         "custom_path/test/support/data_case.ex",
         "Ecto.Adapters.SQL.Sandbox.start_owner"
       )
+
+      assert_file("custom_path/test/custom_path_web/controllers/error_html_test.exs", fn file ->
+        refute file =~ "async: true"
+      end)
+
+      assert_file("custom_path/test/custom_path_web/controllers/error_json_test.exs", fn file ->
+        refute file =~ "async: true"
+      end)
     end)
   end
 

installer/test/phx_new_umbrella_test.exs

@@ -150,8 +150,17 @@ defmodule Mix.Tasks.Phx.New.UmbrellaTest do
       )
 
       assert_file(web_path(@app, "test/#{@app}_web/controllers/page_controller_test.exs"))
-      assert_file(web_path(@app, "test/#{@app}_web/controllers/error_html_test.exs"))
-      assert_file(web_path(@app, "test/#{@app}_web/controllers/error_json_test.exs"))
+
+      assert_file(
+        web_path(@app, "test/#{@app}_web/controllers/error_html_test.exs"),
+        "async: true"
+      )
+
+      assert_file(
+        web_path(@app, "test/#{@app}_web/controllers/error_json_test.exs"),
+        "async: true"
+      )
+
       assert_file(web_path(@app, "test/support/conn_case.ex"))
       assert_file(web_path(@app, "test/test_helper.exs"))
 
@@ -625,6 +634,16 @@ defmodule Mix.Tasks.Phx.New.UmbrellaTest do
       assert_file(root_path(app, "config/runtime.exs"), [~r/url: database_url/])
 
       assert_file(web_path(app, "test/support/conn_case.ex"), "DataCase.setup_sandbox(tags)")
+
+      assert_file(
+        web_path(app, "test/custom_path_web/controllers/error_html_test.exs"),
+        fn file -> refute file =~ "async: true" end
+      )
+
+      assert_file(
+        web_path(app, "test/custom_path_web/controllers/error_json_test.exs"),
+        fn file -> refute file =~ "async: true" end
+      )
     end)
   end
 
@@ -652,6 +671,16 @@ defmodule Mix.Tasks.Phx.New.UmbrellaTest do
 
       assert_file(web_path(app, "test/support/conn_case.ex"), "DataCase.setup_sandbox(tags)")
 
+      assert_file(
+        web_path(app, "test/custom_path_web/controllers/error_html_test.exs"),
+        fn file -> refute file =~ "async: true" end
+      )
+
+      assert_file(
+        web_path(app, "test/custom_path_web/controllers/error_json_test.exs"),
+        fn file -> refute file =~ "async: true" end
+      )
+
       assert_file(root_path(app, ".gitignore"), "*.db")
       assert_file(root_path(app, ".gitignore"), "*.db-*")
     end)
@@ -679,6 +708,16 @@ defmodule Mix.Tasks.Phx.New.UmbrellaTest do
       assert_file(root_path(app, "config/runtime.exs"), [~r/url: database_url/])
 
       assert_file(web_path(app, "test/support/conn_case.ex"), "DataCase.setup_sandbox(tags)")
+
+      assert_file(
+        web_path(app, "test/custom_path_web/controllers/error_html_test.exs"),
+        fn file -> refute file =~ "async: true" end
+      )
+
+      assert_file(
+        web_path(app, "test/custom_path_web/controllers/error_json_test.exs"),
+        fn file -> refute file =~ "async: true" end
+      )
     end)
   end
 
@@ -822,8 +861,8 @@ defmodule Mix.Tasks.Phx.New.UmbrellaTest do
         assert_file("another/lib/another/endpoint.ex", ~r/defmodule Another.Endpoint do/)
 
         assert_file("another/test/another/controllers/page_controller_test.exs")
-        assert_file("another/test/another/controllers/error_html_test.exs")
-        assert_file("another/test/another/controllers/error_json_test.exs")
+        assert_file("another/test/another/controllers/error_html_test.exs", "async: true")
+        assert_file("another/test/another/controllers/error_json_test.exs", "async: true")
         assert_file("another/test/support/conn_case.ex")
         assert_file("another/test/test_helper.exs")