Collect ArgoCD entities for applications, cluster and repos (DataDog#23917)

* test argocd submission

* add unique identifier

* better unique identifier

* allow list approach

* validations

* argocd to submit entity metadata

* Update 23917.added

* remove public plication

* Update metadata.csv

* use a different separator

* pin dcb

* remove review files

* fix naming

* clean up

* more knobs

* nits

* Read genresources config from pydantic and clean up reviewer findings

- Defer ArgocdResourceCollector construction to the first check() call so
  it observes the populated pydantic config (self.config) instead of the
  raw instance dict; drops the hardcoded TTL / interval / cap literals
  that were duplicating the spec defaults.
- Read collect_genresources from self.config.
- Swap the volume-cap log args to read type/fetched/cap in order and use
  log.exception for the fetch-failure path so tracebacks survive.
- Spec wording: ArgoCD (one word) and allowlist / allowlisted per the
  style guide.
- Tests: add a _check() helper that loads the config models and attaches
  the collector, add coverage for the missing-endpoint path and for
  credential scrubbing on cluster.connectionState.message, tighten the
  volume-cap assertion against the new message format, and mark the file
  as pytest.mark.unit.

* Update argocd/datadog_checks/argocd/resources.py

Co-authored-by: Kyle Neale <kyle.neale@datadoghq.com>

* Update argocd/datadog_checks/argocd/resources.py

Co-authored-by: Kyle Neale <kyle.neale@datadoghq.com>

* Update argocd/datadog_checks/argocd/resources.py

Co-authored-by: Kyle Neale <kyle.neale@datadoghq.com>

* Update argocd/datadog_checks/argocd/resources.py

Co-authored-by: Kyle Neale <kyle.neale@datadoghq.com>

---------

Co-authored-by: Kyle-Neale <kyle.neale@datadoghq.com>

Author: steveny91

argocd/assets/configuration/spec.yaml

@@ -51,11 +51,114 @@ files:
     - name: commit_server_endpoint
       description: |
         Endpoint exposing Commit Server's Prometheus metrics.
+      fleet_configurable: false
       value:
         display_default: null
         example: http://argocd-commit-server:8087/metrics
         type: string
+    - name: collect_genresources
+      hidden: true
+      fleet_configurable: false
+      description: |
+        Enable the generic resources pilot collector that ships ArgoCD
+        Applications, Clusters, and Repositories to Datadog as generic
+        resources. Disabled by default.
+      value:
+        type: boolean
+        example: false
+    - name: genresources_endpoint
+      hidden: true
+      fleet_configurable: false
+      description: |
+        Base URL of the ArgoCD REST API (for example, ``https://argocd.example.com``).
+        Required when ``collect_genresources`` is set to ``true``.
+      value:
+        display_default: null
+        example: https://<ARGOCD_HOST>
+        type: string
+    - name: genresources_auth_token
+      hidden: true
+      fleet_configurable: false
+      description: |
+        Raw bearer token used to authenticate against the ArgoCD REST API.
+        When set, the collector adds ``Authorization: Bearer <token>`` to
+        each REST request. Leave unset to inherit the request authentication
+        configured on the instance (for example, the structured ``auth_token``
+        config object handled by the HTTP wrapper).
+      secret: true
+      value:
+        display_default: null
+        example: <BEARER_TOKEN>
+        type: string
+    - name: genresources_ttl_seconds
+      hidden: true
       fleet_configurable: false
+      description: |
+        Time-to-live in seconds applied to every emitted resource.
+        Resources expire ``ttl_seconds`` after the last observation.
+        Minimum of 1.
+      value:
+        type: integer
+        example: 21600
+        minimum: 1
+    - name: genresources_collection_interval_seconds
+      hidden: true
+      fleet_configurable: false
+      description: |
+        Minimum number of seconds between generic resources collection cycles.
+        The collector polls the ArgoCD API at most once per interval,
+        independent of the check's metrics scrape frequency, to limit load on
+        the ArgoCD API server in large deployments. Minimum of 1.
+      value:
+        type: integer
+        example: 120
+        minimum: 1
+    - name: genresources_max_resources_per_cycle
+      hidden: true
+      fleet_configurable: false
+      description: |
+        Maximum number of items emitted per resource type per check cycle.
+        When an ArgoCD API endpoint returns more than this, the excess is
+        dropped and a warning is logged. Applied independently to
+        Applications, Clusters, and Repositories.
+      value:
+        type: integer
+        example: 10000
+        minimum: 1
+    - name: genresources_extra_include_paths
+      hidden: true
+      fleet_configurable: false
+      description: |
+        Additional dotted JSON paths appended to the baseline allowlist of
+        every collected resource type. Only enumerated paths are shipped;
+        anything not listed never leaves the Agent. ``[*]`` denotes array
+        iteration. A path that does not match any field for a given type is
+        silently skipped. This list is additive; it can add fields to ship but
+        cannot remove baseline fields. Each path must resolve to a value, or a
+        list of values, not a whole object: a path that lands on an object
+        causes that resource to be dropped, so enumerate leaf fields rather
+        than their parents.
+      value:
+        type: array
+        items:
+          type: string
+        example: []
+    - name: genresources_exclude_paths
+      hidden: true
+      fleet_configurable: false
+      description: |
+        Dotted JSON paths to remove from the baseline allowlist of every
+        collected resource type, applied after ``genresources_extra_include_paths``.
+        An entry removes any allowlisted path equal to it or nested beneath it;
+        for example, ``status.history`` drops every ``status.history[*]`` field
+        while ``status.conditions[*].message`` drops only that leaf. Use this to
+        stop shipping a field without a code change. Removing fields only ever
+        ships less, never more.
+      value:
+        type: array
+        items:
+          type: string
+        example: []
     - template: instances/openmetrics
       overrides:
         openmetrics_endpoint.required: false

argocd/changelog.d/23917.added

@@ -0,0 +1 @@
+Add a collector that submits ArgoCD applications, clusters, and repositories to Datadog as generic resources.

argocd/datadog_checks/argocd/check.py

@@ -15,6 +15,7 @@
     NOTIFICATIONS_CONTROLLER_METRICS,
     REPO_SERVER_METRICS,
 )
+from .resources import ArgocdResourceCollector
 
 (
     API_SERVER_NAMESPACE,
@@ -40,6 +41,17 @@ def __init__(self, name, init_config, instances):
         super(ArgocdCheck, self).__init__(name, init_config, instances)
         self.check_initializations.appendleft(self.parse_config)
         self.check_initializations.append(self.configure_additional_transformers)
+        self._resource_collector: ArgocdResourceCollector | None = None
+
+    def check(self, instance):
+        if self.config.collect_genresources:
+            if self._resource_collector is None:
+                self._resource_collector = ArgocdResourceCollector(self)
+            try:
+                self._resource_collector.collect()
+            except Exception:
+                self.log.exception("genresources: collection cycle failed")
+        super().check(instance)
 
     def parse_config(self):
         endpoint_configs = [
@@ -97,7 +109,7 @@ def argocd_cluster_connection_status_transformer(_metric, sample_data, _runtime_
         return argocd_cluster_connection_status_transformer
 
     def configure_additional_transformers(self):
-        endpoints = [key for key in self.instance.keys() if "_endpoint" in key]
+        endpoints = [key for key in self.instance.keys() if "_endpoint" in key and self.instance[key] in self.scrapers]
         for endpoint in endpoints:
             if endpoint == "app_controller_endpoint":
                 self.scrapers[self.instance[endpoint]].metric_transformer.add_custom_transformer(

argocd/datadog_checks/argocd/config_models/defaults.py

@@ -36,6 +36,10 @@ def instance_collect_counters_with_distributions():
     return False
 
 
+def instance_collect_genresources():
+    return False
+
+
 def instance_collect_histogram_buckets():
     return True
 
@@ -56,6 +60,18 @@ def instance_enable_legacy_tags_normalization():
     return True
 
 
+def instance_genresources_collection_interval_seconds():
+    return 120
+
+
+def instance_genresources_max_resources_per_cycle():
+    return 10000
+
+
+def instance_genresources_ttl_seconds():
+    return 21600
+
+
 def instance_histogram_buckets_as_distributions():
     return False
 

argocd/datadog_checks/argocd/config_models/instance.py

@@ -101,6 +101,7 @@ class InstanceConfig(BaseModel):
     cache_metric_wildcards: Optional[bool] = None
     cache_shared_labels: Optional[bool] = None
     collect_counters_with_distributions: Optional[bool] = None
+    collect_genresources: Optional[bool] = None
     collect_histogram_buckets: Optional[bool] = None
     commit_server_endpoint: Optional[str] = None
     connect_timeout: Optional[float] = None
@@ -113,6 +114,13 @@ class InstanceConfig(BaseModel):
     exclude_metrics_by_labels: Optional[MappingProxyType[str, Union[bool, tuple[str, ...]]]] = None
     extra_headers: Optional[MappingProxyType[str, Any]] = None
     extra_metrics: Optional[tuple[Union[str, MappingProxyType[str, Union[str, ExtraMetrics]]], ...]] = None
+    genresources_auth_token: Optional[str] = None
+    genresources_collection_interval_seconds: Optional[int] = Field(None, ge=1)
+    genresources_endpoint: Optional[str] = None
+    genresources_exclude_paths: Optional[tuple[str, ...]] = None
+    genresources_extra_include_paths: Optional[tuple[str, ...]] = None
+    genresources_max_resources_per_cycle: Optional[int] = Field(None, ge=1)
+    genresources_ttl_seconds: Optional[int] = Field(None, ge=1)
     headers: Optional[MappingProxyType[str, Any]] = None
     histogram_buckets_as_distributions: Optional[bool] = None
     hostname_format: Optional[str] = None