Add more fields for metadata collection (DataDog#24091)

* cherry pick

* changelog

* lint

Author: steveny91

argocd/changelog.d/24091.added

@@ -0,0 +1 @@
+Add more fields for metadata collection

argocd/datadog_checks/argocd/resources.py

@@ -92,17 +92,32 @@ def _scrub_url_credentials(text: str) -> str:
     return URL_CREDENTIALS_PATTERN.sub(r"\1", text)
 
 
+def _scrub_repo_urls(container: dict) -> None:
+    """Strip credentials from a source's repoURL and from any multi-source repoURLs, in place."""
+    source = container.get("source")
+    if isinstance(source, dict) and isinstance(source.get("repoURL"), str):
+        source["repoURL"] = _strip_url_userinfo(source["repoURL"])
+    for src in container.get("sources") or []:
+        if isinstance(src, dict) and isinstance(src.get("repoURL"), str):
+            src["repoURL"] = _strip_url_userinfo(src["repoURL"])
+
+
 def _sanitize_item(item: dict, resource_type: str) -> None:
     """Strip embedded credentials from repo URLs and free-text messages in place before they ship."""
     if resource_type == "argocd_application":
-        spec = item.get("spec") or {}
-        source = spec.get("source")
-        if isinstance(source, dict) and isinstance(source.get("repoURL"), str):
-            source["repoURL"] = _strip_url_userinfo(source["repoURL"])
-        for src in spec.get("sources") or []:
-            if isinstance(src, dict) and isinstance(src.get("repoURL"), str):
-                src["repoURL"] = _strip_url_userinfo(src["repoURL"])
-        for condition in (item.get("status") or {}).get("conditions") or []:
+        status = item.get("status") or {}
+        _scrub_repo_urls(item.get("spec") or {})
+        for entry in status.get("history") or []:
+            if isinstance(entry, dict):
+                _scrub_repo_urls(entry)
+        operation_state = status.get("operationState")
+        if isinstance(operation_state, dict) and isinstance(operation_state.get("message"), str):
+            operation_state["message"] = _scrub_url_credentials(operation_state["message"])
+        summary = status.get("summary") or {}
+        external_urls = summary.get("externalURLs")
+        if isinstance(external_urls, list):
+            summary["externalURLs"] = [_strip_url_userinfo(u) if isinstance(u, str) else u for u in external_urls]
+        for condition in status.get("conditions") or []:
             if isinstance(condition, dict) and isinstance(condition.get("message"), str):
                 condition["message"] = _scrub_url_credentials(condition["message"])
     elif resource_type == "argocd_repository":

argocd/datadog_checks/argocd/resources_constants.py

@@ -19,6 +19,8 @@
     "paths": (
         "metadata.name",
         "metadata.namespace",
+        "metadata.uid",
+        "metadata.creationTimestamp",
         "spec.project",
         "spec.source.repoURL",
         "spec.source.path",
@@ -37,23 +39,36 @@
         "status.sync.revision",
         "status.health.status",
         "status.health.message",
+        "status.health.lastTransitionTime",
         "status.conditions[*].type",
         "status.conditions[*].message",
         "status.conditions[*].lastTransitionTime",
         "status.operationState.phase",
         "status.operationState.startedAt",
         "status.operationState.finishedAt",
+        "status.operationState.message",
+        "status.operationState.retryCount",
         "status.operationState.operation.initiatedBy.username",
         "status.operationState.operation.initiatedBy.automated",
         "status.sourceType",
         "status.reconciledAt",
         "status.summary.images[*]",
+        "status.summary.externalURLs[*]",
         "status.history[*].id",
         "status.history[*].revision",
         "status.history[*].deployedAt",
         "status.history[*].deployStartedAt",
         "status.history[*].initiatedBy.username",
         "status.history[*].initiatedBy.automated",
+        "status.history[*].source.repoURL",
+        "status.history[*].source.path",
+        "status.history[*].source.targetRevision",
+        "status.history[*].source.chart",
+        "status.history[*].sources[*].repoURL",
+        "status.history[*].sources[*].path",
+        "status.history[*].sources[*].targetRevision",
+        "status.history[*].sources[*].chart",
+        "status.history[*].revisions[*]",
         "status.resources[*].kind",
         "status.resources[*].name",
         "status.resources[*].namespace",
@@ -75,9 +90,12 @@
         "namespaces[*]",
         "connectionState.status",
         "connectionState.message",
+        "connectionState.attemptedAt",
         "info.applicationsCount",
         "info.serverVersion",
         "info.cacheInfo.resourcesCount",
+        "info.connectionState.status",
+        "shard",
     ),
     "map_paths": ("labels",),
     "annotation_keys": (),
@@ -91,6 +109,11 @@
         "project",
         "connectionState.status",
         "connectionState.message",
+        "connectionState.attemptedAt",
+        "insecure",
+        "enableLfs",
+        "enableOCI",
+        "forceHttpBasicAuth",
     ),
     "map_paths": (),
     "annotation_keys": (),

argocd/tests/test_resources.py

@@ -14,6 +14,7 @@
     APPLICATION_INCLUDE,
     CLUSTER_INCLUDE,
     GENRESOURCES_API_UP_METRIC,
+    REPOSITORY_INCLUDE,
 )
 from datadog_checks.dev.http import MockResponse
 
@@ -124,6 +125,115 @@ def test_application_include_contains_kubernetes_resource_identity_for_automatic
     } <= set(APPLICATION_INCLUDE["paths"])
 
 
+def test_application_include_adds_pilot_metadata_and_state_fields():
+    assert {
+        "metadata.uid",
+        "metadata.creationTimestamp",
+        "status.health.lastTransitionTime",
+        "status.operationState.retryCount",
+    } <= set(APPLICATION_INCLUDE["paths"])
+
+
+def test_application_include_contains_deployment_history_and_operation_fields():
+    assert {
+        "status.operationState.message",
+        "status.summary.externalURLs[*]",
+        "status.history[*].source.repoURL",
+        "status.history[*].source.path",
+        "status.history[*].sources[*].repoURL",
+        "status.history[*].revisions[*]",
+    } <= set(APPLICATION_INCLUDE["paths"])
+
+
+def test_collect_scrubs_credentials_from_operation_state_message(mock_http_response_per_endpoint):
+    app = _application("broken")
+    app["status"]["operationState"] = {
+        "phase": "Failed",
+        "message": "sync failed: https://oauth2:t0ken@github.com/org/repo: auth required",
+    }
+    mock_http_response_per_endpoint(
+        {
+            APPLICATIONS_URL: [_items_response([app])],
+            CLUSTERS_URL: [_items_response([])],
+            REPOSITORIES_URL: [_items_response([])],
+        }
+    )
+    check = _check()
+
+    with patch.object(check, "submit_generic_resource") as submit:
+        check._resource_collector.collect()
+
+    app_call = next(c for c in submit.call_args_list if c.kwargs["type"] == "argocd_application")
+    message = app_call.kwargs["fields"]["status"]["operationState"]["message"]
+    assert "t0ken" not in message
+    assert "https://github.com/org/repo" in message
+
+
+def test_collect_strips_credentials_from_external_urls(mock_http_response_per_endpoint):
+    app = _application("web")
+    app["status"]["summary"] = {"externalURLs": ["https://user:t0ken@app.example.com"]}
+    mock_http_response_per_endpoint(
+        {
+            APPLICATIONS_URL: [_items_response([app])],
+            CLUSTERS_URL: [_items_response([])],
+            REPOSITORIES_URL: [_items_response([])],
+        }
+    )
+    check = _check()
+
+    with patch.object(check, "submit_generic_resource") as submit:
+        check._resource_collector.collect()
+
+    app_call = next(c for c in submit.call_args_list if c.kwargs["type"] == "argocd_application")
+    assert app_call.kwargs["fields"]["status"]["summary"]["externalURLs"] == ["https://app.example.com"]
+
+
+def test_real_helper_ships_multisource_history_and_scrubs_its_repo_urls(aggregator, mock_http_response_per_endpoint):
+    # Runs the real helper: proves nested [*] (history[*].sources[*]) projects AND history repoURLs are scrubbed.
+    app = _application("checkout")
+    app["status"]["history"] = [
+        {
+            "id": 1,
+            "source": {"repoURL": "https://oauth2:t0ken@github.com/org/repo", "path": "guestbook"},
+            "sources": [{"repoURL": "https://oauth2:t0ken@github.com/org/multi", "path": "base"}],
+        }
+    ]
+    mock_http_response_per_endpoint(
+        {
+            APPLICATIONS_URL: [_items_response([app])],
+            CLUSTERS_URL: [_items_response([])],
+            REPOSITORIES_URL: [_items_response([])],
+        }
+    )
+    check = _check()
+
+    check._resource_collector.collect()
+
+    payloads = aggregator.get_event_platform_events("genresources", parse_json=False)
+    blob = b"".join(p if isinstance(p, bytes) else p.encode() for p in payloads)
+    assert b"guestbook" in blob  # single-source history field projected
+    assert b"base" in blob  # multi-source history field projected (nested [*] works)
+    assert b"t0ken" not in blob  # both history repoURLs scrubbed before ship
+
+
+def test_cluster_include_contains_connection_and_shard_fields():
+    assert {
+        "connectionState.attemptedAt",
+        "info.connectionState.status",
+        "shard",
+    } <= set(CLUSTER_INCLUDE["paths"])
+
+
+def test_repository_include_contains_connection_and_capability_flags():
+    assert {
+        "connectionState.attemptedAt",
+        "insecure",
+        "enableLfs",
+        "enableOCI",
+        "forceHttpBasicAuth",
+    } <= set(REPOSITORY_INCLUDE["paths"])
+
+
 def test_application_key_uses_app_identity_not_destination(mock_http_response_per_endpoint):
     apps = [
         _application("web", namespace="team-a", cluster="https://remote", dest_namespace="prod"),