Don't apply the FD_SETSIZE check to Windows sockets

On Windows a SOCKET is an opaque handle, not a small index, and an fd_set
is a counted array of sockets rather than a bitmask. libmodbus only ever
puts a single socket in a set, so it can't overflow it.

The fd >= FD_SETSIZE checks added in 7b13341 therefore don't apply on
Windows, where they instead reject any socket whose handle value reaches
FD_SETSIZE (64 by default) - which happens after a few dozen open handles
and breaks Modbus TCP with EINVAL / "exceeds FD_SETSIZE".

Wrap the check in a macro that keeps it on POSIX and disables it on
Windows. POSIX behaviour is unchanged.

Fixes #842

Author: TStal

src/modbus-private.h

@@ -22,6 +22,21 @@ typedef int ssize_t;
 
 #include "modbus.h"
 
+/* select()/FD_SET treat a descriptor as an index into a fixed-size bitmask on
+ * POSIX, so a descriptor greater than or equal to FD_SETSIZE would write past
+ * the end of the fd_set and must be rejected. On Windows a SOCKET is an opaque
+ * handle whose numeric value is unrelated to FD_SETSIZE, and an fd_set is a
+ * counted array of sockets (fd_count + fd_array[]) rather than a bitmask. Since
+ * libmodbus only ever registers a single socket in a set, it can never overflow
+ * fd_array; the POSIX range check therefore does not apply on Windows and, if
+ * kept, wrongly rejects valid sockets whose handle value happens to reach
+ * FD_SETSIZE (64 by default). Restrict the check to non-Windows platforms. */
+#if defined(_WIN32)
+#define MODBUS_FD_OUT_OF_RANGE(fd) (0)
+#else
+#define MODBUS_FD_OUT_OF_RANGE(fd) ((fd) >= FD_SETSIZE)
+#endif
+
 MODBUS_BEGIN_DECLS
 
 /* It's not really the minimal length (the real one is report slave ID

src/modbus-rtu.c

@@ -1239,7 +1239,7 @@ _modbus_rtu_select(modbus_t *ctx, fd_set *rset, struct timeval *tv, int length_t
             }
             /* Necessary after an error */
             FD_ZERO(rset);
-            if (ctx->s < 0 || ctx->s >= FD_SETSIZE) {
+            if (ctx->s < 0 || MODBUS_FD_OUT_OF_RANGE(ctx->s)) {
                 errno = EINVAL;
                 return -1;
             }

src/modbus-tcp.c

@@ -305,7 +305,7 @@ static int _connect(int sockfd,
 
         /* Wait to be available in writing */
         FD_ZERO(&wset);
-        if (sockfd >= FD_SETSIZE) {
+        if (MODBUS_FD_OUT_OF_RANGE(sockfd)) {
             errno = EINVAL;
             return -1;
         }
@@ -363,7 +363,7 @@ static int _modbus_tcp_connect(modbus_t *ctx)
         return -1;
     }
 
-    if (s >= FD_SETSIZE) {
+    if (MODBUS_FD_OUT_OF_RANGE(s)) {
         if (ctx->debug) {
             fprintf(
                 stderr,
@@ -471,7 +471,7 @@ static int _modbus_tcp_pi_connect(modbus_t *ctx)
         if (s < 0)
             continue;
 
-        if (s >= FD_SETSIZE) {
+        if (MODBUS_FD_OUT_OF_RANGE(s)) {
             if (ctx->debug) {
                 fprintf(
                     stderr,
@@ -550,7 +550,7 @@ static int _modbus_tcp_flush(modbus_t *ctx)
         tv.tv_sec = 0;
         tv.tv_usec = 0;
         FD_ZERO(&rset);
-        if (ctx->s < 0 || ctx->s >= FD_SETSIZE) {
+        if (ctx->s < 0 || MODBUS_FD_OUT_OF_RANGE(ctx->s)) {
             errno = EINVAL;
             return -1;
         }
@@ -805,7 +805,7 @@ int modbus_tcp_accept(modbus_t *ctx, int *s)
         return -1;
     }
 
-    if (ctx->s >= FD_SETSIZE) {
+    if (MODBUS_FD_OUT_OF_RANGE(ctx->s)) {
         if (ctx->debug) {
             fprintf(
                 stderr,
@@ -853,7 +853,7 @@ int modbus_tcp_pi_accept(modbus_t *ctx, int *s)
         return -1;
     }
 
-    if (ctx->s >= FD_SETSIZE) {
+    if (MODBUS_FD_OUT_OF_RANGE(ctx->s)) {
         if (ctx->debug) {
             fprintf(
                 stderr,
@@ -890,7 +890,7 @@ _modbus_tcp_select(modbus_t *ctx, fd_set *rset, struct timeval *tv, int length_t
             }
             /* Necessary after an error */
             FD_ZERO(rset);
-            if (ctx->s < 0 || ctx->s >= FD_SETSIZE) {
+            if (ctx->s < 0 || MODBUS_FD_OUT_OF_RANGE(ctx->s)) {
                 errno = EINVAL;
                 return -1;
             }

src/modbus.c

@@ -392,7 +392,7 @@ int _modbus_receive_msg(modbus_t *ctx, uint8_t *msg, msg_type_t msg_type)
 
     /* Add a file descriptor to the set */
     FD_ZERO(&rset);
-    if (ctx->s < 0 || ctx->s >= FD_SETSIZE) {
+    if (ctx->s < 0 || MODBUS_FD_OUT_OF_RANGE(ctx->s)) {
         if (ctx->debug) {
             fprintf(stderr, "ERROR Invalid socket descriptor %d\n", ctx->s);
         }