Add max_retries and cooldown timer to send_hep (#3627)

* Add logic to prevent an unreachable HEP destination on any TCP connection to crash OpenSIPS

* Make the pkg_free into a spearate function

Author: rita7lopes

modules/proto_hep/README

@@ -246,7 +246,34 @@ modparam("proto_hep", "hep_async_max_postponed_chunks", 16)
 modparam("proto_hep", "hep_capture_id", 234)
 ...
 
-1.3.10. hep_async_local_connect_timeout (integer)
+1.3.10. hep_retry_cooldown (integer)
+
+   This parameter defines how many seconds OpenSIPS should wait 
+   before retrying a TCP connection to the HEP destination after 
+   reaching the maximum number of failed attempts set by 
+   hep_max_retries. Limitation: 16-bit integer.
+
+   Default value is "3600".
+
+   Example 1.10. Set hep_retry_cooldown parameter
+...
+modparam("proto_hep", "hep_retry_cooldown", 60)
+...
+
+1.3.11. hep_max_retries (integer)
+
+   This parameter defines the maximum number of attempts 
+   OpenSIPS will make to establish a TCP connection with 
+   the HEP destination. Limitation: 16-bit integer.
+
+   Default value is "5".
+
+   Example 1.11. Set hep_max_retries parameter
+...
+modparam("proto_hep", "hep_max_retries", 10)
+...
+
+1.3.12. hep_async_local_connect_timeout (integer)
 
    If hep_async is enabled, this specifies the number of
    milliseconds that a connect will be tried in blocking mode
@@ -256,12 +283,12 @@ modparam("proto_hep", "hep_capture_id", 234)
 
    Default value is 100 ms.
 
-   Example 1.10. Set hep_async_local_connect_timeout parameter
+   Example 1.12. Set hep_async_local_connect_timeout parameter
 ...
 modparam("proto_hep", "hep_async_local_connect_timeout", 200)
 ...
 
-1.3.11. hep_async_local_write_timeout (integer)
+1.3.13. hep_async_local_write_timeout (integer)
 
    If hep_async is enabled, this specifies the number of
    milliseconds that a write op will be tried in blocking mode
@@ -271,7 +298,7 @@ modparam("proto_hep", "hep_async_local_connect_timeout", 200)
 
    Default value is 10 ms.
 
-   Example 1.11. Set hep_async_local_write_timeout parameter
+   Example 1.13. Set hep_async_local_write_timeout parameter
 ...
 modparam("proto_hep", "hep_async_local_write_timeout", 100)
 ...

modules/proto_hep/doc/proto_hep_admin.xml

@@ -283,6 +283,48 @@ modparam("proto_hep", "hep_capture_id", 234)
 		</example>
 	</section>
 
+	<section id="param_hep_retry_cooldown" xreflabel="hep_retry_cooldown">
+		<title><varname>hep_retry_cooldown</varname> (integer)</title>
+		<para>
+			This parameter defines how many seconds OpenSIPS should wait before retrying a TCP connection to the HEP destination after reaching the maximum number of failed attempts set by hep_max_retries.
+		Limitation: 16-bit integer.
+		</para>
+		<para>
+		<emphasis>
+		Default value is "3600".
+		</emphasis>
+		</para>
+		<example>
+		<title>Set <varname>hep_retry_cooldown</varname> parameter</title>
+		<programlisting format="linespecific">
+...
+modparam("proto_hep", "hep_retry_cooldown", 60)
+...
+</programlisting>
+		</example>
+	</section>
+
+	<section id="param_hep_max_retries" xreflabel="hep_max_retries">
+		<title><varname>hep_max_retries</varname> (integer)</title>
+		<para>
+			This parameter defines the maximum number of attempts OpenSIPS will make to establish a TCP connection with the HEP destination.
+		Limitation: 16-bit integer.
+		</para>
+		<para>
+		<emphasis>
+		Default value is "5".
+		</emphasis>
+		</para>
+		<example>
+		<title>Set <varname>hep_max_retries</varname> parameter</title>
+		<programlisting format="linespecific">
+...
+modparam("proto_hep", "hep_max_retries", 10)
+...
+</programlisting>
+		</example>
+	</section>
+
 	<section id="param_hep_async_local_connect_timeout" xreflabel="hep_async_local_connect_timeout">
 		<title><varname>hep_async_local_connect_timeout</varname> (integer)</title>
 		<para>

modules/proto_hep/hep.c

@@ -51,6 +51,8 @@
 #define HEP_PROTO_SIP  0x01
 
 static int control_id = -1;
+static int hep_failed_retries = 0;
+static time_t hep_last_attempt = 0;
 
 struct hep_message_id {
 	char* proto;
@@ -71,6 +73,8 @@ static hid_list_p hid_list=NULL;
 static hid_list_p *hid_dyn_list=NULL;
 static gen_lock_t *hid_dyn_lock=NULL;
 
+extern int hep_max_retries;
+extern int hep_retry_cooldown;
 extern int hep_capture_id;
 extern int payload_compression;
 extern int homer5_on;
@@ -1677,6 +1681,20 @@ int add_hep_payload(trace_message message, char* pld_name, str* pld_value)
 	return 0;
 }
 
+static void free_hep_send_resources(struct proxy_l *p, union sockaddr_union *to, char *buf) {
+	if (p) {
+		free_proxy(p);
+		pkg_free(p);
+	}
+	if (to) {
+		pkg_free(to);
+	}
+	if (buf) {
+		pkg_free(buf);
+	}
+}
+
+
 
 int send_hep_message(trace_message message, trace_dest dest, const struct socket_info* send_sock)
 {
@@ -1710,35 +1728,46 @@ int send_hep_message(trace_message message, trace_dest dest, const struct socket
 	/* */
 	p=mk_proxy( &hep_dest->ip, hep_dest->port_no ? hep_dest->port_no : HEP_PORT, hep_dest->transport, 0);
 	if (p == NULL) {
-		pkg_free(buf);
 		LM_ERR("bad hep host name!\n");
+		free_hep_send_resources(NULL, NULL, buf);
 		goto end;
 	}
 
 	to=(union sockaddr_union *)pkg_malloc(sizeof(union sockaddr_union));
 	if (to == 0) {
 		LM_ERR("no more pkg mem!\n");
-		pkg_free(buf);
-		free_proxy(p);
-		pkg_free(p);
+		free_hep_send_resources(p, NULL, buf);
 		goto end;
 	}
 
 	hostent2su(to, &p->host, p->addr_idx, p->port?p->port:HEP_PORT);
 
+	time_t now = time(NULL);
+
+	// Check cooldown logic
+	if (hep_failed_retries >= hep_max_retries && (now - hep_last_attempt) < hep_retry_cooldown) {
+		LM_ERR("HEP send suppressed: too many failures (%d), in cooldown (%ld seconds left)\n", hep_failed_retries, hep_retry_cooldown - (now - hep_last_attempt));
+		free_hep_send_resources(p, to, buf);
+		goto end;
+	}
+
+	hep_last_attempt = now;
+
 	do {
 		if (msg_send(send_sock, hep_dest->transport, to, 0, buf, len, NULL) < 0) {
-			LM_ERR("Cannot send hep message!\n");
+			LM_ERR("Cannot send HEP message!\n");
+			hep_failed_retries++;
 			continue;
 		}
-		ret=0;
+
+		// Success: reset retry state
+		hep_failed_retries = 0;
+		ret = 0;
 		break;
-	} while ( get_next_su( p, to, 0)==0);
+	} while (get_next_su(p, to, 0) == 0);
 
-	free_proxy(p);
-	pkg_free(p);
-	pkg_free(to);
-	pkg_free(buf);
+
+	free_hep_send_resources(p, to, buf);
 
 end:
 	return ret;

modules/proto_hep/proto_hep.c

@@ -92,6 +92,8 @@ static int hep_tls_async_handshake_connect_timeout = 10;
 int hep_ctx_idx = 0;
 int hep_capture_id = 1;
 int payload_compression = 0;
+int hep_max_retries = 5;
+int hep_retry_cooldown = 3600; //seconds
 
 int homer5_on = 1;
 str homer5_delim = {":", 0};