Skip to content

Commit 3441d4a

Browse files
vitkyrkaclaude
vitkyrka
and
claude
authored
traffic_server: add container-based config discovery support (DataDog#24488)
* Add container-based config discovery support to traffic_server. Adds a manual from_ports discovery strategy that generates a traffic_server_url candidate against the stats_over_http plugin's /_stats endpoint on the container's discovered port, plus the auto_conf.yaml ad_identifiers entry and E2E discovery tests. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> * Address traffic server discovery review feedback * Fix traffic server discovery E2E assertions * comment --------- Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
1 parent 5f35075 commit 3441d4a

14 files changed

Lines changed: 360 additions & 17 deletions

File tree

‎datadog_checks_dev/changelog.d/24488.added‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Allow config discovery candidates to include literal arrays and mappings.

‎datadog_checks_dev/datadog_checks/dev/tooling/configuration/consumers/model/model_consumer.py‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
# All rights reserved
33
# Licensed under a 3-clause BSD style license (see LICENSE)
44
import warnings
5+
from pprint import pformat
56
from typing import Any, Dict, List, Tuple
67

78
import yaml
@@ -395,15 +396,23 @@ def _emit_strategy_loop(stanza: dict[str, Any], strategy_name: str) -> list[str]
395396
def _emit_candidate_body(candidate: dict[str, Any]) -> list[str]:
396397
"""Emit the model-backed candidate construction for one candidate mapping."""
397398
lines = [' instance_data = {']
398-
for field_name, template in candidate.items():
399-
if '{' in str(template):
400-
rendered = f"{template!r}.format(service=service, **ctx)"
401-
else:
402-
rendered = repr(template)
399+
for field_name, value in candidate.items():
400+
rendered = ModelConsumer._render_candidate_value(value)
403401
lines.append(f' {field_name!r}: {rendered},')
404402
lines.append(' }')
405403
lines.append(' instance = InstanceConfig.model_validate(')
406404
lines.append(" instance_data, context={'configured_fields': frozenset(instance_data)}")
407405
lines.append(" ).model_dump(by_alias=True, mode='json', exclude_none=True)")
408406
lines.append(" yield {'init_config': shared, 'instances': [instance]}")
409407
return lines
408+
409+
@staticmethod
410+
def _render_candidate_value(value: Any) -> str:
411+
"""Render a discovery candidate value as a Python expression."""
412+
if isinstance(value, str):
413+
if '{' in value:
414+
return f'{value!r}.format(service=service, **ctx)'
415+
416+
return repr(value)
417+
418+
return pformat(value, width=120, sort_dicts=False)

‎datadog_checks_dev/datadog_checks/dev/tooling/configuration/spec.py‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -347,11 +347,35 @@ def discovery_validator(discovery: Any, options: list, loader: Any, file_name: s
347347
continue
348348
if instance_option_names and field_name not in instance_option_names:
349349
loader.errors.append(f'{candidate_location}, {field_name}: Not a recognized instance option')
350-
if not isinstance(template, str):
351-
loader.errors.append(f'{candidate_location}, {field_name}: Candidate templates must be strings')
352-
continue
353350

354-
_validate_discovery_template(template, loader, candidate_location, field_name, placeholders)
351+
validate_discovery_candidate_value(template, loader, candidate_location, field_name, placeholders)
352+
353+
354+
def validate_discovery_candidate_value(
355+
value: Any, loader: Any, location: str, field_name: str, placeholders: dict[str, frozenset[str] | None]
356+
) -> None:
357+
if isinstance(value, str):
358+
_validate_discovery_template(value, loader, location, field_name, placeholders)
359+
elif isinstance(value, dict):
360+
validate_discovery_candidate_mapping_keys(value, loader, location, field_name)
361+
362+
363+
def validate_discovery_candidate_mapping_keys(
364+
value: dict[str, Any], loader: Any, location: str, field_name: str
365+
) -> None:
366+
for key, item in value.items():
367+
if not isinstance(key, str):
368+
loader.errors.append(f'{location}, {field_name}: Candidate mapping keys must be strings')
369+
continue
370+
371+
if isinstance(item, dict):
372+
validate_discovery_candidate_mapping_keys(item, loader, location, f'{field_name}.{key}')
373+
elif isinstance(item, list):
374+
for index, nested_item in enumerate(item, 1):
375+
if isinstance(nested_item, dict):
376+
validate_discovery_candidate_mapping_keys(
377+
nested_item, loader, location, f'{field_name}.{key}[{index}]'
378+
)
355379

356380

357381
def _validate_discovery_template(

‎datadog_checks_dev/tests/tooling/configuration/consumers/model/test_discovery.py‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,83 @@ def candidates(service: Service) -> Iterator[dict[str, Any]]:
7575
)
7676

7777

78+
def test_literal_candidate_values():
79+
consumer = get_model_consumer(
80+
"""
81+
name: test
82+
version: 0.0.0
83+
files:
84+
- name: test.yaml
85+
discovery:
86+
strategies:
87+
- strategy: from_ports
88+
port_hints:
89+
- 9090
90+
candidates:
91+
- endpoint: http://{service.host}:{port.number}/m
92+
metric_patterns:
93+
include:
94+
- test.metric.{2}
95+
options:
96+
- template: init_config
97+
options: []
98+
- template: instances
99+
options:
100+
- name: endpoint
101+
description: words
102+
required: true
103+
value:
104+
type: string
105+
- name: metric_patterns
106+
description: words
107+
value:
108+
type: object
109+
additionalProperties: true
110+
"""
111+
)
112+
113+
discovery_contents, discovery_errors = consumer.render()['test.yaml']['discovery.py']
114+
assert not discovery_errors
115+
assert discovery_contents == normalize_yaml(
116+
"""
117+
from __future__ import annotations
118+
119+
from collections.abc import Iterator
120+
from typing import Any
121+
122+
from datadog_checks.base.utils.discovery import Service, candidate_ports
123+
from datadog_checks.test.config_models import discovery_overrides
124+
from datadog_checks.test.config_models.instance import InstanceConfig
125+
from datadog_checks.test.config_models.shared import SharedConfig
126+
127+
128+
def _generated_candidates(service: Service) -> Iterator[dict[str, Any]]:
129+
shared = SharedConfig.model_validate({}, context={'configured_fields': frozenset()}).model_dump(
130+
by_alias=True, mode='json', exclude_none=True
131+
)
132+
# discovery[0]: from_ports
133+
for port in candidate_ports(service, [9090]):
134+
ctx = {'port': port}
135+
instance_data = {
136+
'endpoint': 'http://{service.host}:{port.number}/m'.format(service=service, **ctx),
137+
'metric_patterns': {'include': ['test.metric.{2}']},
138+
}
139+
instance = InstanceConfig.model_validate(
140+
instance_data, context={'configured_fields': frozenset(instance_data)}
141+
).model_dump(by_alias=True, mode='json', exclude_none=True)
142+
yield {'init_config': shared, 'instances': [instance]}
143+
144+
145+
def candidates(service: Service) -> Iterator[dict[str, Any]]:
146+
override = getattr(discovery_overrides, 'candidates', None)
147+
if override is None:
148+
yield from _generated_candidates(service)
149+
else:
150+
yield from override(service, default=_generated_candidates)
151+
"""
152+
)
153+
154+
78155
def test_local_strategy():
79156
consumer = get_model_consumer(
80157
"""

‎datadog_checks_dev/tests/tooling/configuration/test_load.py‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -544,6 +544,44 @@ def test_discovery_candidate_field_cross_check():
544544
) in spec.errors
545545

546546

547+
def test_discovery_candidate_accepts_literal_values():
548+
spec = get_spec(
549+
"""
550+
version: 0.0.0
551+
files:
552+
- name: test.yaml
553+
example_name: test.yaml.example
554+
discovery:
555+
strategies:
556+
- strategy: from_ports
557+
port_hints:
558+
- 9090
559+
candidates:
560+
- endpoint: http://{service.host}:{port.number}/metrics
561+
metric_patterns:
562+
include:
563+
- test.metric.{2}
564+
options:
565+
- template: init_config
566+
- template: instances
567+
options:
568+
- name: endpoint
569+
description: endpoint
570+
required: true
571+
value:
572+
type: string
573+
- name: metric_patterns
574+
description: metric patterns
575+
value:
576+
type: object
577+
additionalProperties: true
578+
"""
579+
)
580+
spec.load()
581+
582+
assert not spec.errors
583+
584+
547585
def test_discovery_local_strategy_accepted():
548586
spec = get_spec(
549587
"""

‎traffic_server/assets/configuration/spec.yaml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,33 @@ name: Traffic Server
22
fleet_configurable: true
33
files:
44
- name: traffic_server.yaml
5+
discovery:
6+
strategies:
7+
- strategy: from_ports
8+
port_hints:
9+
- 8080
10+
candidates:
11+
- traffic_server_url: "http://{service.host}:{port.number}/_stats"
12+
metric_patterns:
13+
include:
14+
- traffic_server.node.*
15+
- traffic_server.process.http.code.*
16+
- traffic_server.process.http(s|2)?.*current
17+
- traffic_server.process.http(s|2)?.*_requests
18+
- traffic_server.process.http(s|2)?.*_connections
19+
- traffic_server.process.http.transaction_.*
20+
- traffic_server.process.hostdb.*
21+
- traffic_server.process.dns.*
22+
- traffic_server.process.traffic_server.memory.rss
23+
- traffic_server.process.eventloop.*
24+
- traffic_server.process.ssl.user_agent_session.*
25+
- traffic_server.process.ssl.ssl_error.*
26+
- traffic_server.process.ssl.user_agent_sessions
27+
- traffic_server.process.cache.total.*
28+
- traffic_server.process.cache.volume.dir.*
29+
- traffic_server.process.cache.volume.percent_full
30+
- traffic_server.process.cache.volume.bytes.*
31+
- traffic_server.process.cache.volume.ram_cache.*
532
options:
633
- template: init_config
734
options:
@@ -65,3 +92,10 @@ files:
6592
- type: file
6693
path: /opt/trafficserver/var/log/trafficserver/error.log
6794
source: traffic_server
95+
- name: auto_conf.yaml
96+
options:
97+
- template: ad_identifiers
98+
overrides:
99+
value.example:
100+
- trafficserver
101+
- template: auto_conf/discovery

‎traffic_server/changelog.d/24488.added‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Add container-based config discovery support.

‎traffic_server/datadog_checks/traffic_server/config_models/discovery.py‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# (C) Datadog, Inc. 2026-present
2+
# All rights reserved
3+
# Licensed under a 3-clause BSD style license (see LICENSE)
4+
5+
# This file is autogenerated.
6+
# To change this file you should edit assets/configuration/spec.yaml and then run the following commands:
7+
# ddev -x validate config -s <INTEGRATION_NAME>
8+
# ddev -x validate models -s <INTEGRATION_NAME>
9+
10+
from __future__ import annotations
11+
12+
from collections.abc import Iterator
13+
from typing import Any
14+
15+
from datadog_checks.base.utils.discovery import Service, candidate_ports
16+
from datadog_checks.traffic_server.config_models import discovery_overrides
17+
from datadog_checks.traffic_server.config_models.instance import InstanceConfig
18+
from datadog_checks.traffic_server.config_models.shared import SharedConfig
19+
20+
21+
def _generated_candidates(service: Service) -> Iterator[dict[str, Any]]:
22+
shared = SharedConfig.model_validate({}, context={'configured_fields': frozenset()}).model_dump(
23+
by_alias=True, mode='json', exclude_none=True
24+
)
25+
# discovery[0]: from_ports
26+
for port in candidate_ports(service, [8080]):
27+
ctx = {'port': port}
28+
instance_data = {
29+
'traffic_server_url': 'http://{service.host}:{port.number}/_stats'.format(service=service, **ctx),
30+
'metric_patterns': {
31+
'include': [
32+
'traffic_server.node.*',
33+
'traffic_server.process.http.code.*',
34+
'traffic_server.process.http(s|2)?.*current',
35+
'traffic_server.process.http(s|2)?.*_requests',
36+
'traffic_server.process.http(s|2)?.*_connections',
37+
'traffic_server.process.http.transaction_.*',
38+
'traffic_server.process.hostdb.*',
39+
'traffic_server.process.dns.*',
40+
'traffic_server.process.traffic_server.memory.rss',
41+
'traffic_server.process.eventloop.*',
42+
'traffic_server.process.ssl.user_agent_session.*',
43+
'traffic_server.process.ssl.ssl_error.*',
44+
'traffic_server.process.ssl.user_agent_sessions',
45+
'traffic_server.process.cache.total.*',
46+
'traffic_server.process.cache.volume.dir.*',
47+
'traffic_server.process.cache.volume.percent_full',
48+
'traffic_server.process.cache.volume.bytes.*',
49+
'traffic_server.process.cache.volume.ram_cache.*',
50+
]
51+
},
52+
}
53+
instance = InstanceConfig.model_validate(
54+
instance_data, context={'configured_fields': frozenset(instance_data)}
55+
).model_dump(by_alias=True, mode='json', exclude_none=True)
56+
yield {'init_config': shared, 'instances': [instance]}
57+
58+
59+
def candidates(service: Service) -> Iterator[dict[str, Any]]:
60+
override = getattr(discovery_overrides, 'candidates', None)
61+
if override is None:
62+
yield from _generated_candidates(service)
63+
else:
64+
yield from override(service, default=_generated_candidates)

‎traffic_server/datadog_checks/traffic_server/config_models/discovery_overrides.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# (C) Datadog, Inc. 2026-present
2+
# All rights reserved
3+
# Licensed under a 3-clause BSD style license (see LICENSE)
4+
5+
# Override the generated discovery candidates() for this integration.
6+
#
7+
# Define a candidates(service, default) function to wrap or replace the generated
8+
# candidate generation. `default` is the generated generator; call it to reuse
9+
# the spec-driven candidates, or ignore it to replace them entirely.
10+
#
11+
# def candidates(service, default):
12+
# yield from default(service)

‎traffic_server/datadog_checks/traffic_server/config_models/discovery_strategies.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# (C) Datadog, Inc. 2026-present
2+
# All rights reserved
3+
# Licensed under a 3-clause BSD style license (see LICENSE)
4+
5+
# Here you can define custom (local:) discovery strategies for this integration.
6+
#
7+
# Decorate a generator with @discovery_strategy (imported from
8+
# datadog_checks.base.utils.discovery) and reference it from the spec discovery
9+
# stanza as `strategy: local:<function_name>`. The function receives the
10+
# discovered Service plus the inputs declared in the spec and yields one context
11+
# (ctx) mapping per candidate, exposing the keys listed in `provides`.
12+
#
13+
# from datadog_checks.base.utils.discovery import discovery_strategy
14+
#
15+
# @discovery_strategy(provides=('svc',))
16+
# def from_some_config(service, config_path):
17+
# ...
18+
# yield {'svc': ...}

0 commit comments

Comments
 (0)