Fix FD_SET overflow when socket fd >= FD_SETSIZE

When a TCP server accepts a socket with a file descriptor >= FD_SETSIZE
(commonly 1024), the FD_SET macro writes past the end of the fd_set
buffer, causing memory corruption or abort via glibc __fdelt_chk.

An attacker can trigger this by exhausting file descriptors so the next
accepted connection gets a high-numbered fd, then sending any request
to drive modbus_receive().

Guard all FD_SET call sites: reject fds >= FD_SETSIZE early at
accept/connect time, and add defensive checks before each FD_SET use
in _modbus_receive_msg, _connect, _modbus_tcp_select, _modbus_tcp_flush,
and _modbus_rtu_select.

Reported-by: Lukas Dresel <lukas@artiphishell.com>
Reported-by: Fabio <team@artiphishell.com>

Author: stephane

src/modbus-rtu.c

@@ -1150,6 +1150,10 @@ _modbus_rtu_select(modbus_t *ctx, fd_set *rset, struct timeval *tv, int length_t
             }
             /* Necessary after an error */
             FD_ZERO(rset);
+            if (ctx->s < 0 || ctx->s >= FD_SETSIZE) {
+                errno = EINVAL;
+                return -1;
+            }
             FD_SET(ctx->s, rset);
         } else {
             return -1;

src/modbus-tcp.c

@@ -297,6 +297,10 @@ static int _connect(int sockfd,
 
         /* Wait to be available in writing */
         FD_ZERO(&wset);
+        if (sockfd >= FD_SETSIZE) {
+            errno = EINVAL;
+            return -1;
+        }
         FD_SET(sockfd, &wset);
         rc = select(sockfd + 1, NULL, &wset, NULL, &tv);
         if (rc < 0) {
@@ -350,6 +354,20 @@ static int _modbus_tcp_connect(modbus_t *ctx)
         return -1;
     }
 
+    if (ctx->s >= FD_SETSIZE) {
+        if (ctx->debug) {
+            fprintf(
+                stderr,
+                "ERROR Socket descriptor %d exceeds FD_SETSIZE (%d)\n",
+                ctx->s,
+                FD_SETSIZE);
+        }
+        close(ctx->s);
+        ctx->s = -1;
+        errno = EINVAL;
+        return -1;
+    }
+
     rc = _modbus_tcp_set_ipv4_options(ctx->s);
     if (rc == -1) {
         close(ctx->s);
@@ -440,6 +458,18 @@ static int _modbus_tcp_pi_connect(modbus_t *ctx)
         if (s < 0)
             continue;
 
+        if (s >= FD_SETSIZE) {
+            if (ctx->debug) {
+                fprintf(
+                    stderr,
+                    "ERROR Socket descriptor %d exceeds FD_SETSIZE (%d)\n",
+                    s,
+                    FD_SETSIZE);
+            }
+            close(s);
+            continue;
+        }
+
         if (ai_ptr->ai_family == AF_INET)
             _modbus_tcp_set_ipv4_options(s);
 
@@ -501,6 +531,10 @@ static int _modbus_tcp_flush(modbus_t *ctx)
         tv.tv_sec = 0;
         tv.tv_usec = 0;
         FD_ZERO(&rset);
+        if (ctx->s < 0 || ctx->s >= FD_SETSIZE) {
+            errno = EINVAL;
+            return -1;
+        }
         FD_SET(ctx->s, &rset);
         rc = select(ctx->s + 1, &rset, NULL, NULL, &tv);
         if (rc == -1) {
@@ -743,6 +777,20 @@ int modbus_tcp_accept(modbus_t *ctx, int *s)
         return -1;
     }
 
+    if (ctx->s >= FD_SETSIZE) {
+        if (ctx->debug) {
+            fprintf(
+                stderr,
+                "ERROR Socket descriptor %d exceeds FD_SETSIZE (%d)\n",
+                ctx->s,
+                FD_SETSIZE);
+        }
+        close(ctx->s);
+        ctx->s = -1;
+        errno = EINVAL;
+        return -1;
+    }
+
     if (ctx->debug) {
         char buf[INET_ADDRSTRLEN];
         if (inet_ntop(AF_INET, &(addr.sin_addr), buf, INET_ADDRSTRLEN) == NULL) {
@@ -777,6 +825,20 @@ int modbus_tcp_pi_accept(modbus_t *ctx, int *s)
         return -1;
     }
 
+    if (ctx->s >= FD_SETSIZE) {
+        if (ctx->debug) {
+            fprintf(
+                stderr,
+                "ERROR Socket descriptor %d exceeds FD_SETSIZE (%d)\n",
+                ctx->s,
+                FD_SETSIZE);
+        }
+        close(ctx->s);
+        ctx->s = -1;
+        errno = EINVAL;
+        return -1;
+    }
+
     if (ctx->debug) {
         char buf[INET6_ADDRSTRLEN];
         if (inet_ntop(AF_INET6, &(addr.sin6_addr), buf, INET6_ADDRSTRLEN) == NULL) {
@@ -800,6 +862,10 @@ _modbus_tcp_select(modbus_t *ctx, fd_set *rset, struct timeval *tv, int length_t
             }
             /* Necessary after an error */
             FD_ZERO(rset);
+            if (ctx->s < 0 || ctx->s >= FD_SETSIZE) {
+                errno = EINVAL;
+                return -1;
+            }
             FD_SET(ctx->s, rset);
         } else {
             return -1;

src/modbus.c

@@ -385,6 +385,13 @@ int _modbus_receive_msg(modbus_t *ctx, uint8_t *msg, msg_type_t msg_type)
 
     /* Add a file descriptor to the set */
     FD_ZERO(&rset);
+    if (ctx->s < 0 || ctx->s >= FD_SETSIZE) {
+        if (ctx->debug) {
+            fprintf(stderr, "ERROR Invalid socket descriptor %d\n", ctx->s);
+        }
+        errno = EINVAL;
+        return -1;
+    }
     FD_SET(ctx->s, &rset);
 
     /* We need to analyse the message step by step.  At the first step, we want