fix: Eliminate flaky parallel test failures caused by port collisions and timing races

src/code.cloudfoundry.org/gorouter/common/component_test.go

@@ -167,8 +167,9 @@ var _ = Describe("Component", func() {
 		var natsRunner *test_util.NATSRunner
 
 		BeforeEach(func() {
-			natsPort := test_util.NextAvailPort()
+			natsPort := test_util.ReservePort()
 			natsRunner = test_util.NewNATSRunner(int(natsPort))
+			test_util.ReleasePort(natsPort)
 			natsRunner.Start()
 			mbusClient = natsRunner.MessageBus
 			mbusClient.Opts.SkipSubjectValidation = true

src/code.cloudfoundry.org/gorouter/handlers/reporter_test.go

@@ -88,7 +88,9 @@ var _ = Describe("Reporter Handler", func() {
 	})
 
 	It("emits routing response metrics", func() {
+		before := time.Now()
 		handler.ServeHTTP(resp, req)
+		after := time.Now()
 
 		Expect(fakeReporter.CaptureBadGatewayCallCount()).To(Equal(0))
 
@@ -102,7 +104,11 @@ var _ = Describe("Reporter Handler", func() {
 		Expect(capturedEndpoint.PrivateInstanceId).To(Equal("id"))
 		Expect(capturedEndpoint.PrivateInstanceIndex).To(Equal("1"))
 		Expect(capturedRespCode).To(Equal(http.StatusTeapot))
-		Expect(startTime).To(BeTemporally("~", time.Now(), 100*time.Millisecond))
+		// ReceivedAt is set to timeNow-1ms where timeNow is captured inside
+		// the handler (between before and after), so the exact bracket is:
+		//   before-1ms <= startTime <= after-1ms
+		Expect(startTime).To(BeTemporally(">=", before.Add(-1*time.Millisecond)))
+		Expect(startTime).To(BeTemporally("<=", after.Add(-1*time.Millisecond)))
 		Expect(latency).To(BeNumerically(">", 0))
 		Expect(latency).To(BeNumerically("<", 10*time.Millisecond))
 

src/code.cloudfoundry.org/gorouter/handlers/requestinfo_test.go

@@ -59,16 +59,17 @@ var _ = Describe("RequestInfoHandler", func() {
 	})
 
 	It("sets RequestInfo with StartTime on the context", func() {
+		before := time.Now()
 		handler.ServeHTTP(resp, req, nextHandler)
 		var contextReq *http.Request
 		Eventually(reqChan).Should(Receive(&contextReq))
-
-		expectedStartTime := time.Now()
+		after := time.Now()
 
 		ri, err := handlers.ContextRequestInfo(contextReq)
 		Expect(err).ToNot(HaveOccurred())
 		Expect(ri).ToNot(BeNil())
-		Expect(ri.ReceivedAt).To(BeTemporally("~", expectedStartTime, 10*time.Millisecond))
+		Expect(ri.ReceivedAt).To(BeTemporally(">=", before))
+		Expect(ri.ReceivedAt).To(BeTemporally("<=", after))
 
 	})
 })

src/code.cloudfoundry.org/gorouter/integration/common_integration_test.go

@@ -63,15 +63,17 @@ func (s *testState) SetOnlyTrustClientCACertsTrue() {
 
 func NewTestState() *testState {
 	// TODO: don't hide so much behind these test_util methods
-	cfg, clientTLSConfig := test_util.SpecSSLConfig(test_util.NextAvailPort(), test_util.NextAvailPort(), test_util.NextAvailPort(), test_util.NextAvailPort(), test_util.NextAvailPort(), test_util.NextAvailPort(), test_util.NextAvailPort())
+	// Use ReservePort to keep listeners open until the gorouter process
+	// starts, preventing other processes from grabbing these ports.
+	cfg, clientTLSConfig := test_util.SpecSSLConfig(test_util.ReservePort(), test_util.ReservePort(), test_util.ReservePort(), test_util.ReservePort(), test_util.ReservePort(), test_util.ReservePort(), test_util.ReservePort())
 	cfg.SkipSSLValidation = false
 	cfg.RouteServicesHairpinning = false
 	cfg.CipherString = "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384"
 
 	// TODO: why these magic numbers?
 	cfg.PruneStaleDropletsInterval = 2 * time.Second
 	cfg.DropletStaleThreshold = 10 * time.Second
-	cfg.StartResponseDelayInterval = 1 * time.Second
+	cfg.StartResponseDelayInterval = 0
 	cfg.EndpointTimeout = 15 * time.Second
 	cfg.EndpointDialTimeout = 500 * time.Millisecond
 	cfg.DrainTimeout = 200 * time.Millisecond
@@ -258,6 +260,10 @@ func (s *testState) registerAndWait(rm mbus.RegistryMessage) {
 func (s *testState) StartGorouter() *Session {
 	Expect(s.cfg).NotTo(BeNil(), "set up test cfg before calling this function")
 
+	// Release NATS port first so the NATS server can bind it, while keeping
+	// the other ports reserved until the gorouter starts.
+	test_util.ReleasePort(s.cfg.Nats.Hosts[0].Port)
+
 	s.natsRunner = test_util.NewNATSRunner(int(s.cfg.Nats.Hosts[0].Port))
 	s.natsRunner.Start()
 
@@ -271,6 +277,10 @@ func (s *testState) StartGorouter() *Session {
 	Expect(err).ToNot(HaveOccurred())
 	Expect(os.WriteFile(cfgFile, cfgBytes, 0644)).To(Succeed())
 
+	// Release remaining reserved ports just before the gorouter process
+	// starts, minimizing the TOCTOU window between release and bind.
+	test_util.ReleaseAllPorts()
+
 	cmd := exec.Command(gorouterPath, "-c", cfgFile)
 	s.gorouterSession, err = Start(cmd, GinkgoWriter, GinkgoWriter)
 	Expect(err).ToNot(HaveOccurred())
@@ -297,6 +307,12 @@ func (s *testState) StartGorouterOrFail() {
 }
 
 func (s *testState) StopAndCleanup() {
+	// Stop router before NATS to prevent subscriber's ClosedCB from
+	// firing log.Fatal → os.Exit(1), which kills the test proc.
+	if s.gorouterSession != nil && s.gorouterSession.ExitCode() == -1 {
+		Eventually(s.gorouterSession.Terminate(), 5).Should(Exit(0))
+	}
+
 	if s.natsRunner != nil {
 		s.natsRunner.Stop()
 	}
@@ -308,10 +324,6 @@ func (s *testState) StopAndCleanup() {
 
 	os.RemoveAll(s.tmpdir)
 
-	if s.gorouterSession != nil && s.gorouterSession.ExitCode() == -1 {
-		Eventually(s.gorouterSession.Terminate(), 5).Should(Exit(0))
-	}
-
 	if s.fakeMetron != nil {
 		s.StopMetron()
 	}

src/code.cloudfoundry.org/gorouter/integration/gdpr_test.go

@@ -65,7 +65,7 @@ var _ = Describe("GDPR", func() {
 			testState.EnableAccessLog()
 			testState.cfg.Status.Pass = "pass"
 			testState.cfg.Status.User = "user"
-			testState.cfg.Status.Routes.Port = 6705
+			testState.cfg.Status.Routes.Port = test_util.ReservePort()
 			testState.cfg.Logging.DisableLogForwardedFor = true
 			testState.StartGorouterOrFail()
 
@@ -136,7 +136,7 @@ var _ = Describe("GDPR", func() {
 			testState.EnableAccessLog()
 			testState.cfg.Status.Pass = "pass"
 			testState.cfg.Status.User = "user"
-			testState.cfg.Status.Routes.Port = 6706
+			testState.cfg.Status.Routes.Port = test_util.ReservePort()
 			testState.cfg.Logging.DisableLogSourceIP = true
 			testState.StartGorouterOrFail()
 

src/code.cloudfoundry.org/gorouter/integration/main_test.go

@@ -65,29 +65,33 @@ var _ = Describe("Router Integration", func() {
 		Expect(err).ToNot(HaveOccurred())
 		cfgFile = filepath.Join(tmpdir, "config.yml")
 
-		statusPort = test_util.NextAvailPort()
-		statusTLSPort = test_util.NextAvailPort()
-		statusRoutesPort = test_util.NextAvailPort()
-		proxyPort = test_util.NextAvailPort()
-		natsPort = test_util.NextAvailPort()
-		sslPort = test_util.NextAvailPort()
-		routeServiceServerPort = test_util.NextAvailPort()
+		statusPort = test_util.ReservePort()
+		statusTLSPort = test_util.ReservePort()
+		statusRoutesPort = test_util.ReservePort()
+		proxyPort = test_util.ReservePort()
+		natsPort = test_util.ReservePort()
+		sslPort = test_util.ReservePort()
+		routeServiceServerPort = test_util.ReservePort()
 
 		natsRunner = test_util.NewNATSRunner(int(natsPort))
+		test_util.ReleasePort(natsPort)
 		natsRunner.Start()
 		oauthServerURL = oauthServer.Addr()
 	})
 
 	AfterEach(func() {
+		test_util.ReleaseAllPorts()
+		// Stop router before NATS to prevent subscriber's ClosedCB from
+		// firing log.Fatal → os.Exit(1), which kills the test proc.
+		if gorouterSession != nil && gorouterSession.ExitCode() == -1 {
+			stopGorouter(gorouterSession)
+		}
+
 		if natsRunner != nil {
 			natsRunner.Stop()
 		}
 
 		os.RemoveAll(tmpdir)
-
-		if gorouterSession != nil && gorouterSession.ExitCode() == -1 {
-			stopGorouter(gorouterSession)
-		}
 	})
 
 	Context("when config is invalid", func() {
@@ -609,6 +613,7 @@ var _ = Describe("Router Integration", func() {
 			tempCfg.Logging.MetronAddress = ""
 			writeConfig(tempCfg, cfgFile)
 
+			test_util.ReleaseAllPorts()
 			gorouterCmd := exec.Command(gorouterPath, "-c", cfgFile)
 			gorouterSession, _ = Start(gorouterCmd, GinkgoWriter, GinkgoWriter)
 			Eventually(gorouterSession, 5*time.Second).Should(Exit(1))
@@ -635,7 +640,7 @@ var _ = Describe("Router Integration", func() {
 
 		BeforeEach(func() {
 			testState = NewTestState()
-			testState.cfg.DebugAddr = fmt.Sprintf("127.0.0.1:%d", test_util.NextAvailPort())
+			testState.cfg.DebugAddr = fmt.Sprintf("127.0.0.1:%d", test_util.ReservePort())
 			testState.StartGorouterOrFail()
 			gorouterSession = testState.gorouterSession
 
@@ -1047,7 +1052,7 @@ var _ = Describe("Router Integration", func() {
 	Describe("prometheus metrics", func() {
 		It("starts a prometheus https server", func() {
 			c := createConfig(statusPort, statusTLSPort, statusRoutesPort, proxyPort, routeServiceServerPort, cfgFile, defaultPruneInterval, defaultPruneThreshold, 0, false, 0, natsPort)
-			metricsPort := test_util.NextAvailPort()
+			metricsPort := test_util.ReservePort()
 			serverCAPath, serverCertPath, serverKeyPath, clientCert := tls_helpers.GenerateCaAndMutualTlsCerts()
 
 			c.Prometheus.Enabled = true
@@ -1421,6 +1426,7 @@ var _ = Describe("Router Integration", func() {
 				It("does not exit", func() {
 					writeConfig(cfg, cfgFile)
 
+					test_util.ReleaseAllPorts()
 					gorouterCmd := exec.Command(gorouterPath, "-c", cfgFile)
 					session, err := Start(gorouterCmd, GinkgoWriter, GinkgoWriter)
 					Expect(err).ToNot(HaveOccurred())
@@ -1436,6 +1442,7 @@ var _ = Describe("Router Integration", func() {
 				It("gorouter exits with non-zero code", func() {
 					writeConfig(cfg, cfgFile)
 
+					test_util.ReleaseAllPorts()
 					gorouterCmd := exec.Command(gorouterPath, "-c", cfgFile)
 					session, err := Start(gorouterCmd, GinkgoWriter, GinkgoWriter)
 					Expect(err).ToNot(HaveOccurred())
@@ -1453,6 +1460,7 @@ var _ = Describe("Router Integration", func() {
 					routingApiServer.Close()
 					writeConfig(cfg, cfgFile)
 
+					test_util.ReleaseAllPorts()
 					gorouterCmd := exec.Command(gorouterPath, "-c", cfgFile)
 					session, err := Start(gorouterCmd, GinkgoWriter, GinkgoWriter)
 					Expect(err).ToNot(HaveOccurred())
@@ -1468,6 +1476,7 @@ var _ = Describe("Router Integration", func() {
 				cfg.OAuth.Port = 0
 				writeConfig(cfg, cfgFile)
 
+				test_util.ReleaseAllPorts()
 				gorouterCmd := exec.Command(gorouterPath, "-c", cfgFile)
 				session, err := Start(gorouterCmd, GinkgoWriter, GinkgoWriter)
 				Expect(err).ToNot(HaveOccurred())

src/code.cloudfoundry.org/gorouter/integration/nats_test.go

@@ -36,27 +36,31 @@ var _ = Describe("NATS Integration", func() {
 		Expect(err).ToNot(HaveOccurred())
 		cfgFile = filepath.Join(tmpdir, "config.yml")
 
-		statusPort = test_util.NextAvailPort()
-		statusTLSPort = test_util.NextAvailPort()
-		statusRoutesPort = test_util.NextAvailPort()
-		proxyPort = test_util.NextAvailPort()
-		natsPort = test_util.NextAvailPort()
-		routeServiceServerPort = test_util.NextAvailPort()
+		statusPort = test_util.ReservePort()
+		statusTLSPort = test_util.ReservePort()
+		statusRoutesPort = test_util.ReservePort()
+		proxyPort = test_util.ReservePort()
+		natsPort = test_util.ReservePort()
+		routeServiceServerPort = test_util.ReservePort()
 
 		natsRunner = test_util.NewNATSRunner(int(natsPort))
+		test_util.ReleasePort(natsPort)
 		natsRunner.Start()
 	})
 
 	AfterEach(func() {
+		test_util.ReleaseAllPorts()
+		// Stop router before NATS to prevent subscriber's ClosedCB from
+		// firing log.Fatal → os.Exit(1), which kills the test proc.
+		if gorouterSession != nil && gorouterSession.ExitCode() == -1 {
+			stopGorouter(gorouterSession)
+		}
+
 		if natsRunner != nil {
 			natsRunner.Stop()
 		}
 
 		os.RemoveAll(tmpdir)
-
-		if gorouterSession != nil && gorouterSession.ExitCode() == -1 {
-			stopGorouter(gorouterSession)
-		}
 	})
 
 	It("has Nats connectivity", func() {
@@ -162,7 +166,7 @@ var _ = Describe("NATS Integration", func() {
 		)
 
 		BeforeEach(func() {
-			natsPort2 = test_util.NextAvailPort()
+			natsPort2 = test_util.ReservePort()
 			natsRunner2 = test_util.NewNATSRunner(int(natsPort2))
 
 			pruneInterval = 2 * time.Second
@@ -206,6 +210,7 @@ var _ = Describe("NATS Integration", func() {
 			time.Sleep(heartbeatInterval * 2)
 
 			natsRunner.Stop()
+			test_util.ReleasePort(natsPort2)
 			natsRunner2.Start()
 
 			// Give router time to make a bad decision (i.e. prune routes)
@@ -222,7 +227,7 @@ var _ = Describe("NATS Integration", func() {
 		Context("when suspend_pruning_if_nats_unavailable enabled", func() {
 
 			BeforeEach(func() {
-				natsPort2 = test_util.NextAvailPort()
+				natsPort2 = test_util.ReservePort()
 				natsRunner2 = test_util.NewNATSRunner(int(natsPort2))
 
 				pruneInterval = 200 * time.Millisecond

src/code.cloudfoundry.org/gorouter/integration/test_utils_test.go

@@ -46,7 +46,7 @@ func configDrainSetup(cfg *config.Config, pruneInterval, pruneThreshold time.Dur
 	// as part of pausing
 	cfg.PruneStaleDropletsInterval = pruneInterval
 	cfg.DropletStaleThreshold = pruneThreshold
-	cfg.StartResponseDelayInterval = 1 * time.Second
+	cfg.StartResponseDelayInterval = 0
 	cfg.EndpointTimeout = 5 * time.Second
 	cfg.EndpointDialTimeout = 500 * time.Millisecond
 	cfg.DrainTimeout = 200 * time.Millisecond
@@ -60,6 +60,7 @@ func writeConfig(cfg *config.Config, cfgFile string) {
 }
 
 func startGorouterSession(cfgFile string) *Session {
+	test_util.ReleaseAllPorts()
 	gorouterCmd := exec.Command(gorouterPath, "-c", cfgFile)
 	session, err := Start(gorouterCmd, GinkgoWriter, GinkgoWriter)
 	Expect(err).ToNot(HaveOccurred())

src/code.cloudfoundry.org/gorouter/logger/logger.go

@@ -1,6 +1,7 @@
 package logger
 
 import (
+	"fmt"
 	"io"
 	"log/slog"
 	"os"
@@ -235,6 +236,8 @@ via os.Exit(1) after writing the log message.
 */
 func Fatal(logger *slog.Logger, message string, slogAttrs ...any) {
 	logger.Error(message, slogAttrs...)
+	// Write to stderr so the message survives os.Exit (stderr is unbuffered).
+	fmt.Fprintf(os.Stderr, "FATAL: %s %v\n", message, slogAttrs)
 	os.Exit(1)
 }
 

src/code.cloudfoundry.org/gorouter/mbus/subscriber_test.go

@@ -38,9 +38,9 @@ var _ = Describe("Subscriber", func() {
 	)
 
 	BeforeEach(func() {
-		natsPort = test_util.NextAvailPort()
-
+		natsPort = test_util.ReservePort()
 		natsRunner = test_util.NewNATSRunner(int(natsPort))
+		test_util.ReleasePort(natsPort)
 		natsRunner.Start()
 		natsClient = natsRunner.MessageBus
 
@@ -60,13 +60,13 @@ var _ = Describe("Subscriber", func() {
 	})
 
 	AfterEach(func() {
-		if natsRunner != nil {
-			natsRunner.Stop()
-		}
 		if process != nil {
 			process.Signal(os.Interrupt)
 		}
 		process = nil
+		if natsRunner != nil {
+			natsRunner.Stop()
+		}
 	})
 
 	It("exits when signaled", func() {