Add test to reproduce Invalid server response: 19

.gitignore

@@ -126,3 +126,5 @@ maven-plugin/target/surefire/
 /docs/router-conf.xsd
 .vscode/
 /core/derby.log
+/distribution/conf/apis.yaml
+/distribution/conf/membrane.log

annot/pom.xml

@@ -23,7 +23,7 @@
 		<groupId>org.membrane-soa</groupId>
 		<artifactId>service-proxy-parent</artifactId>
 		<relativePath>../pom.xml</relativePath>
-		<version>6.3.11-SNAPSHOT</version>
+		<version>6.3.17-SNAPSHOT</version>
 	</parent>
 
 	<dependencies>

core/.factorypath

@@ -1,3 +1,3 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?><factorypath>
-    <factorypathentry enabled="true" id="M2_REPO/org/membrane-soa/service-proxy-annot/6.3.11-SNAPSHOT/service-proxy-annot-6.3.11-SNAPSHOT.jar" kind="VARJAR" runInBatchMode="false"/>
+    <factorypathentry enabled="true" id="M2_REPO/org/membrane-soa/service-proxy-annot/6.3.17-SNAPSHOT/service-proxy-annot-6.3.17-SNAPSHOT.jar" kind="VARJAR" runInBatchMode="false"/>
 </factorypath>

core/pom.xml

@@ -23,7 +23,7 @@
         <groupId>org.membrane-soa</groupId>
         <artifactId>service-proxy-parent</artifactId>
         <relativePath>../pom.xml</relativePath>
-        <version>6.3.11-SNAPSHOT</version>
+        <version>6.3.17-SNAPSHOT</version>
     </parent>
 
     <!-- Open Telemetry -->

core/src/main/java/com/predic8/membrane/core/exchange/AbstractExchange.java

@@ -18,6 +18,7 @@
 import com.predic8.membrane.core.http.*;
 import com.predic8.membrane.core.interceptor.*;
 import com.predic8.membrane.core.interceptor.Interceptor.*;
+import com.predic8.membrane.core.interceptor.balancer.ExchangeNodeStatusTracker;
 import com.predic8.membrane.core.model.*;
 import com.predic8.membrane.core.proxies.Proxy;
 import com.predic8.membrane.core.proxies.*;
@@ -28,7 +29,9 @@
 import java.text.*;
 import java.util.*;
 
+import static com.predic8.membrane.core.exchange.Exchange.TRACK_NODE_STATUS;
 import static com.predic8.membrane.core.exchange.ExchangeState.*;
+import static java.lang.Boolean.TRUE;
 
 public abstract class AbstractExchange {
 	private static final Logger log = LoggerFactory.getLogger(AbstractExchange.class.getName());
@@ -68,6 +71,8 @@ public abstract class AbstractExchange {
 
 	private ArrayList<Interceptor> interceptorStack = new ArrayList<>(10);
 
+	private ExchangeNodeStatusTracker nodeStatusTracker;
+
 	private int estimatedHeapSize = -1;
 
 	public AbstractExchange() {}
@@ -408,6 +413,7 @@ public int resetHeapSizeEstimation() {
 
 	protected int estimateHeapSize() {
 		return 2600 +
+				(nodeStatusTracker != null ? nodeStatusTracker.estimateHeapSize() : 0) +
 				(originalRequestUri != null ? originalRequestUri.length() : 0) +
 				(request != null ? request.estimateHeapSize() : 0) +
 				(response != null ? response.estimateHeapSize() : 0);
@@ -436,6 +442,7 @@ public static <T extends AbstractExchange> T updateCopy(T source, T copy, Runnab
 		copy.setRemoteAddr(source.getRemoteAddr());
 		copy.setRemoteAddrIp(source.getRemoteAddrIp());
 		copy.setInterceptorStack(new ArrayList<>(source.getInterceptorStack()));
+		copy.setNodeStatusTracker(source.getNodeStatusTracker() == null ? null : source.getNodeStatusTracker().clone());
 
 		return copy;
 	}
@@ -480,4 +487,16 @@ public Message getMessage(Flow flow) {
 			return request;
 		return response; // RESPONSE and ABORT flow both work on the response
 	}
+
+	public ExchangeNodeStatusTracker getNodeStatusTracker() {
+		return nodeStatusTracker;
+	}
+
+	public void setNodeStatusTracker(ExchangeNodeStatusTracker nodeStatusTracker) {
+		this.nodeStatusTracker = nodeStatusTracker;
+	}
+
+	public void initNodeStatusTracker() {
+		this.nodeStatusTracker = new ExchangeNodeStatusTracker(destinations.size());
+	}
 }

core/src/main/java/com/predic8/membrane/core/exchange/Exchange.java

@@ -54,9 +54,6 @@ public class Exchange extends AbstractExchange {
 
     private Connection targetConnection;
 
-    private int[] nodeStatusCodes;
-
-    private Exception[] nodeExceptions;
 
     private long id;
 
@@ -158,22 +155,6 @@ public Map<String, String> getStringProperties() {
                 .collect(toMap(Map.Entry::getKey, e -> (String) e.getValue()));
     }
 
-    public void setNodeStatusCode(int tryCounter, int code) {
-        if (nodeStatusCodes == null) {
-            nodeStatusCodes = new int[getDestinations().size()];
-        }
-        nodeStatusCodes[tryCounter % getDestinations().size()] = code;
-    }
-
-    public void trackNodeException(int tryCounter, Exception e) {
-        if (!TRUE.equals(properties.get(TRACK_NODE_STATUS)))
-            return;
-        if (nodeExceptions == null) {
-            nodeExceptions = new Exception[getDestinations().size()];
-        }
-        nodeExceptions[tryCounter % getDestinations().size()] = e;
-    }
-
     @Override
     public void detach() {
         super.detach();
@@ -200,14 +181,6 @@ public void setId(long id) {
         this.id = id;
     }
 
-    public int[] getNodeStatusCodes() {
-        return nodeStatusCodes;
-    }
-
-    public Exception[] getNodeExceptions() {
-        return nodeExceptions;
-    }
-
     public String getInboundProtocol() {
         Proxy rule = getProxy();
         if (!(rule instanceof SSLableProxy sp))

core/src/main/java/com/predic8/membrane/core/interceptor/DispatchingInterceptor.java

@@ -57,7 +57,7 @@ public Outcome handleRequest(Exchange exc) {
             try {
                 exc.getDestinations().add(getForwardingDestination(exc));
             } catch (URISyntaxException e) {
-                ProblemDetails pd = user(false, "invalid-path")
+                ProblemDetails pd = user(getRouter().isProduction(), "invalid-path")
                         .title("Invalid request path")
                         .detail(getMessageForURISyntaxException(exc, e))
                         .internal("path", exc.getRequest().getUri());
@@ -109,12 +109,12 @@ protected String getAddressFromTargetElement(Exchange exc) throws MalformedURLEx
 
         if (p.getTargetURL() != null) {
             String targetURL = p.getTarget().compileUrl(exc, REQUEST);
-            if (targetURL.startsWith("http")) {
+            if (targetURL.startsWith("http") || targetURL.startsWith("internal")) {
                 String basePath = UriUtil.getPathFromURL(router.getUriFactory(), targetURL);
-                if (basePath.isEmpty() || "/".equals(basePath)) {
-                    URL base = new URL(targetURL);
+                if (basePath == null || basePath.isEmpty() || "/".equals(basePath)) {
+                    URI base = new URI(targetURL);
                     // Resolve and normalize slashes consistently with the branch below.
-                    return new URL(base, getUri(exc)).toString();
+                    return base.resolve(getUri(exc)).toString();
                 }
             }
             return targetURL;

core/src/main/java/com/predic8/membrane/core/interceptor/administration/AdminPageBuilder.java

@@ -903,6 +903,19 @@ protected void creatExchangeMetaTable(String id) {
         end();
     }
 
+    protected void createExchangeNodeStatusTable(String id) {
+        table().attr("cellpadding", "0", "cellspacing", "0", "border", "0",
+                "class", "display", "id", id);
+        thead();
+        tr();
+        createThs("Node", "Result");
+        end();
+        end();
+        tbody();
+        end();
+        end();
+    }
+
     protected void creatHeaderTable(String id) {
         table().attr("cellpadding", "0", "cellspacing", "0", "border", "0",
                 "class", "display", "id", id);

core/src/main/java/com/predic8/membrane/core/interceptor/administration/AdminRESTInterceptor.java

@@ -225,7 +225,7 @@ public Response getExchange(QueryParameter params, String relativeRootPath) thro
 			return Response.notFound().build();
 		}
 
-		return json(gen -> writeExchange(exc, gen));
+		return json(gen -> writeExchange(exc, gen, true));
 	}
 
 	@Mapping("/admin/rest/exchanges(/?\\?.*)?")
@@ -241,7 +241,7 @@ public Response getExchanges(QueryParameter params, String relativeRootPath) thr
 			gen.writeStartObject();
 			gen.writeArrayFieldStart("exchanges");
 			for (AbstractExchange e : res.getExchanges()) {
-				writeExchange(e, gen);
+				writeExchange(e, gen, false);
 			}
 			gen.writeEndArray();
 			gen.writeNumberField("total", res.getCount());
@@ -250,7 +250,7 @@ public Response getExchanges(QueryParameter params, String relativeRootPath) thr
 		});
 	}
 
-	private void writeExchange(AbstractExchange exc, JsonGenerator gen) throws IOException {
+	private void writeExchange(AbstractExchange exc, JsonGenerator gen, boolean detail) throws IOException {
 		gen.writeStartObject();
 		gen.writeNumberField("id", exc.getId());
 		if (exc.getResponse() != null) {
@@ -310,9 +310,48 @@ private void writeExchange(AbstractExchange exc, JsonGenerator gen) throws IOExc
 
 		gen.writeNumberField("duration", exc.getTimeResReceived() - exc.getTimeReqSent());
 		gen.writeStringField("msgFilePath", JDBCUtil.getFilePath(exc));
+
+		if (detail)
+			writeExchangeDetailFields(exc, gen);
+
 		gen.writeEndObject();
 	}
 
+	private static void writeExchangeDetailFields(AbstractExchange exc, JsonGenerator gen) throws IOException {
+		List<String> destinations = exc.getDestinations();
+		if (destinations != null) {
+			gen.writeArrayFieldStart("destinations");
+			for (String destination : destinations) {
+				if (destination == null)
+					gen.writeNull();
+				else
+					gen.writeString(destination);
+			}
+			gen.writeEndArray();
+		}
+		if (exc.getNodeStatusTracker() != null) {
+			int[] nodeStatusCodes = exc.getNodeStatusTracker().getNodeStatusCodes();
+			if (nodeStatusCodes != null) {
+				gen.writeArrayFieldStart("nodeStatusCodes");
+				for (int statusCode : nodeStatusCodes) {
+					gen.writeNumber(statusCode);
+				}
+				gen.writeEndArray();
+			}
+			Exception[] nodeExceptions = exc.getNodeStatusTracker().getNodeExceptions();
+			if (nodeExceptions != null) {
+				gen.writeArrayFieldStart("nodeExceptions");
+				for (Exception ex : nodeExceptions) {
+					if (ex == null || ex.getMessage() == null)
+						gen.writeNull();
+					else
+						gen.writeString(ex.getMessage());
+				}
+				gen.writeEndArray();
+			}
+		}
+	}
+
 	public static String getClientAddr(boolean useXForwardedForAsClientAddr, AbstractExchange exc) {
         return AdminApiInterceptor.getClientAddr(useXForwardedForAsClientAddr, exc);
 	}

core/src/main/java/com/predic8/membrane/core/interceptor/administration/DynamicAdminPageInterceptor.java

@@ -771,6 +771,10 @@ protected void createTabContent() {
 				end();
 				div().id("tab1");
 				creatExchangeMetaTable("request-meta");
+				div().id("node-status").style("display:none");
+				h3().text("Node Status").end();
+				createExchangeNodeStatusTable("request-node-status");
+				end();
 				h3().text("Content").end();
 				div().align("right").a().id("request-download-button").text("Download").end().end();
 				div().id("requestContentTab");

core/src/main/java/com/predic8/membrane/core/interceptor/balancer/BalancerHealthMonitor.java

@@ -28,6 +28,8 @@
 import org.springframework.context.*;
 
 import java.io.*;
+import java.net.InetSocketAddress;
+import java.net.Socket;
 import java.util.*;
 
 import static com.predic8.membrane.core.http.Request.*;
@@ -40,7 +42,9 @@
  * Periodically checks the health of all clusters registered
  * on the router and updates each {@link Node}'s status accordingly.
  * When initialized, it schedules a task to call each {@link Node}'s health
- * endpoint and marks nodes as {@link Status#UP} or {@link Status#DOWN} based on the HTTP response.
+ * endpoint and marks nodes as {@link Status#UP} or {@link Status#DOWN} based on the result:
+ * If a health URL is configured for the node, it performs an HTTP request against that endpoint.
+ * Otherwise, it performs a TCP check against the node's host and port.
  * This ensures the load balancer always has up-to-date status for routing decisions.
  * @example <a href="https://github.com/membrane/api-gateway/tree/master/distribution/examples/loadbalancing/6-health-monitor">health monitor example</a>
  * @topic 4. Monitoring, Logging and Statistics
@@ -113,13 +117,16 @@ public void run() {
     }
 
     private Status isHealthy(Node node) {
-        String url = getNodeHealthEndpoint(node);
-        try {
-            return getStatus(node, doCall(url));
-        } catch (Exception e) {
-            log.warn("Calling health endpoint failed: {}, {}", url, e.getMessage());
-            return DOWN;
+        String healthUrl = node.getHealthUrl();
+        if(healthUrl != null) {
+            try {
+                return getStatus(node, doCall(healthUrl));
+            } catch (Exception e) {
+                log.warn("Calling health endpoint failed: {}, {}", healthUrl, e.getMessage());
+                return DOWN;
+            }
         }
+        return tcpHealthy(node);
     }
 
     private static Status getStatus(Node node, Exchange exc) {
@@ -140,10 +147,18 @@ private static Status getStatus(Node node, Exchange exc) {
         return UP;
     }
 
-    private static String getNodeHealthEndpoint(Node node) {
-        return node.getHealthUrl() != null
-                ? node.getHealthUrl()
-                : getUrl(node.getHost(), node.getPort());
+    public Status tcpHealthy(Node node) {
+        final String host = node.getHost();
+        final int port = node.getPort();
+
+        try(Socket socket = new Socket()) {
+            socket.connect(new InetSocketAddress(host, port), httpClientConfig.getConnection().getTimeout());
+            log.debug("Node {}:{} is healthy", host, port);
+            return UP;
+        } catch (Exception e) {
+            log.warn("TCP health check failed for {}:{} - {}", host, port, e);
+            return DOWN;
+        }
     }
 
     private Exchange doCall(String url) throws Exception {