feat: Add adaptive poll rate interval logic

Signed-off-by: Cagri Yonca <cagri@ibm.com>

Author: CagriYonca

src/instana/agent/host.py

@@ -315,7 +315,7 @@ def report_data_payload(
 
     def report_metrics(self, payload: Dict[str, Any]) -> Optional[Response]:
         metrics = payload.get("metrics", [])
-        if len(metrics) > 0:
+        if len(metrics) > 0 and len(metrics.get("plugins", [])) > 0:
             metric_bundle = metrics["plugins"][0]["data"]
             response = self.client.post(
                 self.__data_url(),

src/instana/collector/base.py

@@ -49,6 +49,9 @@ def __init__(self, agent: Type["BaseAgent"]) -> None:
         # How often to report snapshot data (in seconds)
         self.snapshot_data_interval = 300
 
+        # Timestamp in seconds of the last time we sent metrics data
+        self.metrics_data_last_sent = 0
+
         # List of helpers that help out in data collection
         self.helpers = []
 
@@ -58,6 +61,7 @@ def __init__(self, agent: Type["BaseAgent"]) -> None:
         self.background_report_lock = threading.RLock()
 
         # Reporting interval for the background thread(s)
+        # Default is 1 but can be changed by the agent options
         self.report_interval = 1
 
         # Flag to indicate if start/shutdown state

src/instana/collector/host.py

@@ -72,29 +72,46 @@ def should_send_snapshot_data(self) -> bool:
         delta = int(time()) - self.snapshot_data_last_sent
         return delta > self.snapshot_data_interval
 
+    def should_send_metrics(self) -> bool:
+        """
+        Determines if metrics data should be sent based on poll_rate.
+        """
+        poll_rate = 1
+        if hasattr(self.agent, "options") and hasattr(self.agent.options, "poll_rate"):
+            poll_rate = self.agent.options.poll_rate
+
+        delta = int(time()) - self.metrics_data_last_sent
+        return delta >= poll_rate
+
     def prepare_payload(self) -> DefaultDict[Any, Any]:
         payload = DictionaryOfStan()
         payload["spans"] = []
         payload["profiles"] = []
         payload["metrics"]["plugins"] = []
 
         try:
+            # Always collect and send spans immediately (every 1 second)
             if not self.span_queue.empty():
                 payload["spans"] = format_span(self.queued_spans())
 
             if not self.profile_queue.empty():
                 payload["profiles"] = self.queued_profiles()
 
-            with_snapshot = self.should_send_snapshot_data()
+            # Only collect metrics based on poll_rate interval
+            if self.should_send_metrics():
+                with_snapshot = self.should_send_snapshot_data()
+
+                plugins = []
+                for helper in self.helpers:
+                    plugins.extend(helper.collect_metrics(with_snapshot=with_snapshot))
 
-            plugins = []
-            for helper in self.helpers:
-                plugins.extend(helper.collect_metrics(with_snapshot=with_snapshot))
+                payload["metrics"]["plugins"] = plugins
 
-            payload["metrics"]["plugins"] = plugins
+                if with_snapshot is True:
+                    self.snapshot_data_last_sent = int(time())
 
-            if with_snapshot is True:
-                self.snapshot_data_last_sent = int(time())
+                # Update metrics last sent timestamp
+                self.metrics_data_last_sent = int(time())
         except Exception:
             logger.debug("non-fatal prepare_payload:", exc_info=True)
 

src/instana/options.py

@@ -351,12 +351,15 @@ class StandardOptions(BaseOptions):
 
     AGENT_DEFAULT_HOST = "localhost"
     AGENT_DEFAULT_PORT = 42699
+    DEFAULT_POLL_RATE = 1
+    MAX_POLL_RATE = 5
 
     def __init__(self, **kwds: Dict[str, Any]) -> None:
         super(StandardOptions, self).__init__()
 
         self.agent_host = os.environ.get("INSTANA_AGENT_HOST", self.AGENT_DEFAULT_HOST)
         self.agent_port = os.environ.get("INSTANA_AGENT_PORT", self.AGENT_DEFAULT_PORT)
+        self.poll_rate = self.DEFAULT_POLL_RATE
 
         if not isinstance(self.agent_port, int):
             self.agent_port = int(self.agent_port)
@@ -506,6 +509,34 @@ def set_disable_tracing(self, tracing_config: Sequence[Dict[str, Any]]) -> None:
             self.disabled_spans.extend(disabled_spans)
             self.enabled_spans.extend(enabled_spans)
 
+    def set_poll_rate(self, plugin_config: Dict[str, Any]) -> None:
+        """Set poll rate from agent plugin configuration."""
+        poll_rate_value = plugin_config.get("poll_rate")
+        if poll_rate_value is None:
+            return
+
+        try:
+            poll_rate = int(poll_rate_value)
+        except (ValueError, TypeError):
+            logger.debug(
+                f"Invalid poll_rate type, defaulting to {self.DEFAULT_POLL_RATE}"
+            )
+            self.poll_rate = self.DEFAULT_POLL_RATE
+            return
+
+        if poll_rate in (self.DEFAULT_POLL_RATE, self.MAX_POLL_RATE):
+            self.poll_rate = poll_rate
+            logger.debug(
+                f"Poll rate set to {self.poll_rate} seconds from agent configuration"
+            )
+            return
+
+        logger.debug(
+            f"Invalid poll_rate value {poll_rate}, defaulting to "
+            f"{self.DEFAULT_POLL_RATE}"
+        )
+        self.poll_rate = self.DEFAULT_POLL_RATE
+
     def set_from(self, res_data: Dict[str, Any]) -> None:
         """
         Set the source identifiers given to use by the Instana Host agent.
@@ -516,13 +547,19 @@ def set_from(self, res_data: Dict[str, Any]) -> None:
             logger.debug(f"options.set_from: Wrong data type - {type(res_data)}")
             return
 
+        # Extract poll_rate from plugin.python.poll_rate
+        if "plugin" in res_data and isinstance(res_data["plugin"], dict):
+            python_plugin = res_data["plugin"].get("python")
+            if isinstance(python_plugin, dict):
+                self.set_poll_rate(python_plugin)
+
         if "secrets" in res_data:
             self.set_secrets(res_data["secrets"])
 
         if "tracing" in res_data:
             self.set_tracing(res_data["tracing"])
-
         else:
+            # Rely on extra headers if no tracing configuration comes from the agent
             if "extraHeaders" in res_data:
                 self.set_extra_headers(res_data["extraHeaders"])