[#728] Reject TCP self-connects in replication connect paths (#729)

Author: vharseko

opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServer.java

@@ -13,6 +13,7 @@
  *
  * Copyright 2006-2010 Sun Microsystems, Inc.
  * Portions Copyright 2011-2016 ForgeRock AS.
+ * Portions Copyrighted 2026 3A Systems, LLC.
  */
 package org.opends.server.replication.server;
 
@@ -21,6 +22,7 @@
 import static org.opends.server.util.StaticUtils.*;
 
 import java.io.IOException;
+import java.net.ConnectException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.ServerSocket;
@@ -414,6 +416,15 @@ private boolean connect(HostPort remoteServerAddress, DN baseDN)
       }
       int timeoutMS = MultimasterReplication.getConnectionTimeoutMS();
       socket.connect(remoteServerAddress.toInetSocketAddress(), timeoutMS);
+      if (isSelfConnection(socket))
+      {
+        // While the remote RS is down, the kernel may pick its port as the
+        // local port of this connecting socket (TCP simultaneous open),
+        // "connecting" it to itself. Keeping such a socket open would hold
+        // the port and prevent the RS from binding it on restart.
+        throw new ConnectException("Connection to " + remoteServerAddress
+            + " is a TCP self-connect, no replication server is listening");
+      }
       session = replSessionSecurity.createClientSession(socket, timeoutMS);
 
       ReplicationServerHandler rsHandler = new ReplicationServerHandler(

opendj-server-legacy/src/main/java/org/opends/server/replication/service/ReplicationBroker.java

@@ -13,7 +13,7 @@
  *
  * Copyright 2006-2010 Sun Microsystems, Inc.
  * Portions Copyright 2011-2016 ForgeRock AS.
- * Portions Copyright 2023-2025 3A Systems LLC.
+ * Portions Copyright 2023-2026 3A Systems LLC.
  * Portions Copyright 2025 Wren Security.
  */
 package org.opends.server.replication.service;
@@ -1083,6 +1083,15 @@ private ConnectedRS performPhaseOneHandshake(String serverURL, boolean keepSessi
       }
       int timeoutMS = MultimasterReplication.getConnectionTimeoutMS();
       socket.connect(HostPort.valueOf(serverURL).toInetSocketAddress(), timeoutMS);
+      if (isSelfConnection(socket))
+      {
+        // While the RS is down, the kernel may pick the RS port as the local
+        // port of this reconnecting socket (TCP simultaneous open),
+        // "connecting" it to itself. Keeping such a socket open would hold
+        // the RS port and prevent the RS from binding it on restart.
+        throw new ConnectException("Connection to " + serverURL
+            + " is a TCP self-connect, no replication server is listening");
+      }
       newSession = replSessionSecurity.createClientSession(socket, timeoutMS);
       boolean isSslEncryption = replSessionSecurity.isSslEncryption();
 

opendj-server-legacy/src/main/java/org/opends/server/util/StaticUtils.java

@@ -13,6 +13,7 @@
  *
  * Copyright 2006-2010 Sun Microsystems, Inc.
  * Portions Copyright 2011-2016 ForgeRock AS.
+ * Portions Copyrighted 2026 3A Systems, LLC.
  */
 package org.opends.server.util;
 
@@ -1319,7 +1320,25 @@ public static boolean isAddressInUse(
     return true;
   }
 
-
+  /**
+   * Indicates whether the provided connected socket is connected to itself,
+   * i.e. its local and remote endpoints are identical.
+   * <p>
+   * Connecting to a local port from the TCP ephemeral range while nothing
+   * listens on it can make the kernel pick that very port as the local port
+   * of the connecting socket: TCP simultaneous open then "establishes" the
+   * connection to itself (observed on Linux). Such a socket occupies the
+   * listen port its target service is about to bind, so callers retrying
+   * connections to a temporarily stopped service must detect and close it.
+   *
+   * @param socket a connected socket
+   * @return true if the socket is connected to itself
+   */
+  public static boolean isSelfConnection(Socket socket)
+  {
+    return socket.getLocalPort() == socket.getPort()
+        && socket.getLocalAddress().equals(socket.getInetAddress());
+  }
 
   /**
    * Returns a lower-case string representation of a given string, verifying for null input string.

opendj-server-legacy/src/test/java/org/opends/server/util/TestStaticUtils.java

@@ -13,6 +13,7 @@
  *
  * Copyright 2006-2009 Sun Microsystems, Inc.
  * Portions Copyright 2013-2016 ForgeRock AS.
+ * Portions Copyrighted 2026 3A Systems, LLC.
  */
 package org.opends.server.util;
 
@@ -26,6 +27,9 @@
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.io.RandomAccessFile;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.nio.channels.FileLock;
@@ -36,6 +40,7 @@
 import org.forgerock.opendj.ldap.ByteString;
 import org.opends.server.TestCaseUtils;
 import org.testng.Assert;
+import org.testng.SkipException;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
@@ -1046,4 +1051,53 @@ public void testStackTraceHasCause() throws Exception
         new RuntimeException(new IllegalThreadStateException()), IllegalArgumentException.class);
     Assert.assertTrue(hasCause, "Third case : IllegalThreadStateException should be detected as a cause");
   }
+
+  @Test
+  public void testIsSelfConnectionFalseForNormalConnection() throws Exception
+  {
+    try (ServerSocket listener = new ServerSocket())
+    {
+      listener.bind(new InetSocketAddress("127.0.0.1", 0));
+      try (Socket client = new Socket())
+      {
+        client.connect(listener.getLocalSocketAddress(), 2000);
+        try (Socket accepted = listener.accept())
+        {
+          Assert.assertFalse(StaticUtils.isSelfConnection(client));
+          Assert.assertFalse(StaticUtils.isSelfConnection(accepted));
+        }
+      }
+    }
+  }
+
+  @Test
+  public void testIsSelfConnectionTrueForSelfConnectedSocket() throws Exception
+  {
+    // find a free port
+    final int port;
+    try (ServerSocket tmp = new ServerSocket())
+    {
+      tmp.bind(new InetSocketAddress("127.0.0.1", 0));
+      port = tmp.getLocalPort();
+    }
+
+    // Force the TCP self-connect (simultaneous open) that the kernel can
+    // produce spontaneously when connecting to an unbound port from the
+    // ephemeral range: bind the local end to the very port being connected.
+    try (Socket socket = new Socket())
+    {
+      socket.setReuseAddress(true);
+      socket.bind(new InetSocketAddress("127.0.0.1", port));
+      try
+      {
+        socket.connect(new InetSocketAddress("127.0.0.1", port), 2000);
+      }
+      catch (IOException e)
+      {
+        // BSD-based stacks (macOS) refuse an explicit self-connect
+        throw new SkipException("TCP self-connect not supported by this OS: " + e);
+      }
+      Assert.assertTrue(StaticUtils.isSelfConnection(socket));
+    }
+  }
 }