Flux CD: Add kube_state_metrics endpoint config, add metric fluxcd.gotk.resource.info, and update the Flux CD overview dashboard (#24053)

* Add fluxcd.gotk.resource.info

* Update dashboard to use resource.info (Flux 2.1+) while keeping legacy reconcile.condition / suspend.status panels

* Reference metrics are deprecated in metadata.csv

* Remove addition to README

* Add changelog

* Correct dashboard titles and update changelog name

* Fix language mentioning deprecration

* Update fluxcd.json based on feedback

Co-authored-by: DeForest Richards <56796055+drichards-87@users.noreply.github.com>

* update changelog

* Add config kube_state_metrics_endpoint to collect kube-state-metrics

* Add code path for KSM metrics

* Add test for KSM metrics

* Fix lint issue

* Fix dashboard titles

* Update changelog to include kube_state_metrics_endpoint config option

* Fix queries to use lowercase values

* Fix dashboard note and widget titles

---------

Co-authored-by: DeForest Richards <56796055+drichards-87@users.noreply.github.com>

Author: ddog-nasirthomas

fluxcd/assets/configuration/spec.yaml

@@ -9,3 +9,9 @@ files:
   - template: instances
     options:
     - template: instances/openmetrics
+    - name: kube_state_metrics_endpoint
+      description: |
+        URL exposing a kube-state-metrics endpoint. Enables collection of kube-state-metrics
+        for Flux custom resources (Flux 2.1+).
+      value:
+        type: string

fluxcd/assets/dashboards/fluxcd.json

@@ -0,0 +1 @@
+Add `fluxcd.gotk.resource.info` metric and `kube_state_metrics_endpoint` configuration option for Flux 2.1+ kube-state-metrics monitoring.

fluxcd/changelog.d/24053.added

@@ -3,12 +3,23 @@
 # Licensed under a 3-clause BSD style license (see LICENSE)
 from datadog_checks.base import OpenMetricsBaseCheckV2
 from datadog_checks.fluxcd.config_models import ConfigMixin
-from datadog_checks.fluxcd.metrics import METRIC_MAP
+from datadog_checks.fluxcd.metrics import KSM_METRICS, METRIC_MAP
 
 
 class FluxcdCheck(OpenMetricsBaseCheckV2, ConfigMixin):
     __NAMESPACE__ = "fluxcd"
     DEFAULT_METRIC_LIMIT = 0
 
+    def __init__(self, name, init_config, instances):
+        super().__init__(name, init_config, instances)
+        self.check_initializations.appendleft(self._parse_config)
+
+    def _parse_config(self):
+        ksm_endpoint = self.instance.get("kube_state_metrics_endpoint")
+        if ksm_endpoint:
+            self.scraper_configs.append(
+                {**self.instance, "openmetrics_endpoint": ksm_endpoint, "metrics": [KSM_METRICS]}
+            )
+
     def get_default_config(self):
         return {"metrics": [METRIC_MAP]}

fluxcd/datadog_checks/fluxcd/check.py

@@ -123,6 +123,7 @@ class InstanceConfig(BaseModel):
     kerberos_hostname: Optional[str] = None
     kerberos_keytab: Optional[str] = None
     kerberos_principal: Optional[str] = None
+    kube_state_metrics_endpoint: Optional[str] = None
     log_requests: Optional[bool] = None
     metric_patterns: Optional[MetricPatterns] = None
     metrics: Optional[tuple[Union[str, MappingProxyType[str, Union[str, Metrics]]], ...]] = None

fluxcd/datadog_checks/fluxcd/config_models/instance.py

@@ -640,3 +640,9 @@ instances:
     #   - <INCLUDE_REGEX>
     #   exclude:
     #   - <EXCLUDE_REGEX>
+
+    ## @param kube_state_metrics_endpoint - string - optional
+    ## URL exposing a kube-state-metrics endpoint. Enables collection of kube-state-metrics
+    ## for Flux custom resources (Flux 2.1+).
+    #
+    # kube_state_metrics_endpoint: <KUBE_STATE_METRICS_ENDPOINT>

fluxcd/datadog_checks/fluxcd/data/conf.yaml.example

@@ -15,6 +15,10 @@
 }
 assert sorted(FLUX_V1_METRICS) == list(FLUX_V1_METRICS)
 
+KSM_METRICS = {
+    "gotk_resource_info": "gotk.resource.info",
+}
+
 FLUX_V2_METRICS = {
     "leader_election_master_status": "leader_election_master_status",
     "process_cpu_seconds": "process.cpu_seconds",

fluxcd/datadog_checks/fluxcd/metrics.py

@@ -10,6 +10,7 @@ fluxcd.gotk.reconcile.condition,gauge,,,,The current condition status of a GitOp
 fluxcd.gotk.reconcile.duration.seconds.bucket,count,,,,Bucket of the duration in seconds of a GitOps Toolkit resource reconciliation.,0,fluxcd,reconcile duration seconds,
 fluxcd.gotk.reconcile.duration.seconds.count,count,,,,Count of the duration in seconds of a GitOps Toolkit resource reconciliation.,0,fluxcd,reconcile duration seconds,
 fluxcd.gotk.reconcile.duration.seconds.sum,count,,second,,Sum of the duration in seconds of a GitOps Toolkit resource reconciliation.,0,fluxcd,reconcile duration seconds,
+fluxcd.gotk.resource.info,gauge,,,,Flux custom resource state from kube-state-metrics (gotk_resource_info). Use with Flux 2.1+ monitoring.,0,fluxcd,gotk resource info,
 fluxcd.gotk.suspend.status,gauge,,,,The current suspend status of a GitOps Toolkit resource [Deprecated since 2.1.0].,0,fluxcd,gotk suspend status,
 fluxcd.leader_election_master_status,gauge,,,,"Gauge of if the reporting system is master of the relevant lease, 0 indicates backup, 1 indicates master. 'name' is the string used to identify the lease. Make sure to group by name.",0,fluxcd,,
 fluxcd.process.cpu_seconds.count,count,,second,,Total user and system CPU time spent in seconds.,0,fluxcd,,

fluxcd/metadata.csv

@@ -45,3 +45,7 @@
         "fluxcd.workqueue.unfinished_work",
     },
 }
+
+KSM_METRICS = {
+    "fluxcd.gotk.resource.info",
+}

fluxcd/tests/common.py

@@ -102,3 +102,21 @@ def mock_metrics_v2():
         ),
     ):
         yield
+
+
+@pytest.fixture()
+def mock_metrics_ksm():
+    fixture_file = os.path.join(os.path.dirname(__file__), "fixtures", "metrics-ksm.txt")
+
+    with open(fixture_file, "r") as f:
+        content = f.read()
+
+    with mock.patch(
+        "requests.Session.get",
+        return_value=mock.MagicMock(
+            status_code=200,
+            iter_lines=lambda **kwargs: content.split("\n"),
+            headers={"Content-Type": "text/plain"},
+        ),
+    ):
+        yield