test: negative pgrst_db_pool_available in metrics

Proves the failure on #4622.

This doesn't require additional test infra, only nginx. Taking advantage
of the `stream {}` context which is also compatible with unix socket
besides TCP.

Author: steve-chavez

nix/tools/tests.nix

@@ -9,6 +9,7 @@
 , hpc-codecov
 , jq
 , lib
+, nginx
 , postgrest
 , python3
 , runtimeShell
@@ -93,6 +94,7 @@ let
         args = [ "ARG_LEFTOVERS([pytest arguments])" ];
         workingDir = "/";
         withEnv = postgrest.env;
+        withPath = [ nginx ];
       }
       ''
         ${cabal-install}/bin/cabal v2-build ${devCabalOptions} exe:postgrest
@@ -155,6 +157,7 @@ let
         redirectTixFiles = false;
         withEnv = postgrest.env;
         withTmpDir = true;
+        withPath = [ nginx ];
       }
       (
         # required for `hpc markup` in CI; glibcLocales is not available e.g. on Darwin

test/io/config.py

@@ -10,6 +10,7 @@
 FIXTURES = yaml.load(
     (BASEDIR / "fixtures/fixtures.yaml").read_text(), Loader=yaml.Loader
 )
+NGINX_BIN = shutil.which("nginx")
 POSTGREST_BIN = shutil.which("postgrest")
 SECRET = "reallyreallyreallyreallyverysafe"
 

test/io/nginx/nginx.conf

@@ -0,0 +1,13 @@
+# the PG* variables are replaced by preprocessing, not done by nginx itself
+daemon off;
+pid        ./nginx.pid;
+
+events {}
+
+stream {
+  server {
+    listen unix:$PGPROXYHOST/.s.PGSQL.5432;
+    proxy_timeout $PGPROXY_TIMEOUT;
+    proxy_pass unix:$PGHOST/.s.PGSQL.5432;
+  }
+}

test/io/postgrest.py

@@ -8,12 +8,13 @@
 import subprocess
 import tempfile
 import time
+import string
 import urllib.parse
 
 import requests
 import requests_unixsocket
 
-from config import POSTGREST_BIN, hpctixfile
+from config import POSTGREST_BIN, NGINX_BIN, hpctixfile
 
 
 def sleep_until_postgrest_scache_reload():
@@ -170,6 +171,52 @@ def run(
                 process.wait()
 
 
+@contextlib.contextmanager
+def run_pgproxy(env=None, proxy_timeout="1s"):
+    "Run nginx as a unix socket proxy for PostgreSQL and expose PGPROXYHOST."
+    env = dict(os.environ if env is None else env)
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        # build a <tmpdir>/conf/ so `nginx -p` picks the config automatically
+        tmpdir = pathlib.Path(tmpdir)
+        conf_dir = tmpdir / "conf"
+        conf_dir.mkdir(parents=True)
+
+        nginx_env = dict(env)
+        nginx_env["PGPROXYHOST"] = str(tmpdir)
+        nginx_env["PGPROXY_TIMEOUT"] = proxy_timeout
+
+        source_conf = pathlib.Path("test/io/nginx/nginx.conf")
+        out_conf = conf_dir / "nginx.conf"
+        out_conf.write_text(
+            string.Template(source_conf.read_text()).substitute(nginx_env)
+        )
+
+        process = subprocess.Popen(
+            [NGINX_BIN, "-p", str(tmpdir), "-e", "stderr"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            env=nginx_env,
+        )
+
+        if process.poll() is not None:
+            (_, stderr_output) = process.communicate(timeout=1)
+            raise RuntimeError(
+                f"{NGINX_BIN} exited with {process.returncode}: {stderr_output}"
+            )
+
+        try:
+            yield str(tmpdir)
+        finally:
+            process.terminate()
+            try:
+                process.wait(timeout=1)
+            except subprocess.TimeoutExpired:
+                process.kill()
+                process.wait()
+
+
 def freeport(used_ports=None):
     "Find an unused free port on localhost."
     while True:

test/io/test_io.py

@@ -14,6 +14,7 @@
     is_ipv6,
     reset_statement_timeout,
     run,
+    run_pgproxy,
     set_statement_timeout,
     sleep_until_postgrest_config_reload,
     sleep_until_postgrest_full_reload,
@@ -2178,3 +2179,27 @@ def test_vary_default_header_set(defaultenv):
         response = postgrest.session.get("/projects")
 
         assert response.headers["Vary"] == "Accept, Prefer, Range"
+
+
+@pytest.mark.xfail(
+    reason="pgrst_db_pool_available should not go negative on pg network failures",
+    strict=True,
+)
+def test_positive_pool_metric(defaultenv):
+    "When a network failure is caused on the pg connection, pgrst_db_pool_available stays positive"
+
+    with run_pgproxy(defaultenv, proxy_timeout="10ms") as pgproxyhost:
+        env = {**defaultenv, "PGHOST": pgproxyhost}
+
+        with run(env=env, wait_for_readiness=False) as postgrest:
+            time.sleep(2)
+
+            response = postgrest.admin.get("/metrics", timeout=1)
+            assert response.status_code == 200
+
+            metrics = float(
+                re.search(
+                    r"pgrst_db_pool_available (-?\d+(?:\.\d+)?)", response.text
+                ).group(1)
+            )
+            assert metrics >= 0