test(io): add test_so_reuseport_zero_downtime_handover

Author: mkleczek

test/io/postgrest.py

@@ -85,6 +85,7 @@ def run(
     stdin=None,
     env=None,
     port=None,
+    admin_port=None,
     host=None,
     wait_for_readiness=True,
     wait_max_seconds=1,
@@ -113,7 +114,7 @@ def run(
             env["PGRST_SERVER_UNIX_SOCKET"] = str(socketfile)
             baseurl = "http+unix://" + urllib.parse.quote_plus(str(socketfile))
 
-        adminport = freeport(used_ports=[port])
+        adminport = freeport(used_ports=[port]) if admin_port is None else admin_port
         env["PGRST_ADMIN_SERVER_PORT"] = str(adminport)
         adminhost = f"[{host}]" if host and is_ipv6(host) else localhost
         adminurl = f"http://{adminhost}:{adminport}"
@@ -176,10 +177,10 @@ def freeport(used_ports=None):
                 return port
 
 
-def wait_until_exit(postgrest):
+def wait_until_exit(postgrest, timeout=1):
     "Wait for PostgREST to exit, or times out"
     try:
-        return postgrest.process.wait(timeout=1)
+        return postgrest.process.wait(timeout=timeout)
     except subprocess.TimeoutExpired:
         raise PostgrestTimedOut()
 

test/io/test_io.py

@@ -152,6 +152,73 @@ def test_random_port_bound(defaultenv):
         assert True  # liveness check is done by run(), so we just need to check that it doesn't fail
 
 
+@pytest.mark.xfail(reason="PostgREST should not start on a used port", strict=True)
+def test_so_reuseport_zero_downtime_handover(defaultenv):
+    "A second PostgREST instance should take over on the same main/admin ports without request failures."
+
+    # set host to _all_ addresses to force port conflict without SO_REUSEPORT
+    # setting to localhost (which is the default)
+    # might allow running multiple instances on the same port
+    # as the name might be resolved to many IP addresses
+    host = "0.0.0.0"
+    port = freeport()
+    admin_port = freeport(used_ports=[port])
+    failures = []
+    # mutable location shared between threads
+    keep_running = {"value": True}
+
+    # 1. Start first PostgREST instance
+    # 2. Start a "client" thread issuing requests in a loop
+    #    remembering all received errors
+    # 3. Start second PostgREST instance on the same port as the first one
+    # 4. Wait a little and terminate the first instance
+    #
+    # We expect the client does not get any errors after stopping the first instance
+    # and seamlessly migrate to the second instance.
+    #
+    # 5. Stop client thread
+    # 6. Stop second PostgREST instance
+    # 7. Verify client did not get any errors
+    with run(
+        env={**defaultenv},
+        port=port,
+        host=host,
+        admin_port=admin_port,
+    ) as first:
+
+        def continuously_request():
+            while keep_running["value"]:
+                try:
+                    response = first.session.get("/projects", timeout=1)
+                    assert response.status_code == 200
+                except Exception as exc:
+                    failures.append(exc)
+                    break
+                time.sleep(0.2)
+
+        requester = Thread(target=continuously_request)
+        requester.start()
+
+        try:
+            time.sleep(1)
+            with run(
+                env={**defaultenv},
+                port=port,
+                host=host,
+                admin_port=admin_port,
+            ):
+                time.sleep(1)
+                first.process.terminate()
+                wait_until_exit(first, 2)
+
+                time.sleep(1)
+        finally:
+            keep_running["value"] = False
+            requester.join()
+
+    assert failures == []
+
+
 def test_app_settings_reload(tmp_path, defaultenv):
     "App settings should be reloaded from file when PostgREST is sent SIGUSR2."
     config = (CONFIGSDIR / "sigusr2-settings.config").read_text()