Fix apply_overrides to support nested template overrides in spec.yaml (#23348)

* Fix apply_overrides to support nested template overrides in spec.yaml

When a spec.yaml template reference has an overrides: block targeting a field
that lives inside a nested template (e.g. tags inside instances/default ->
instances/all_integrations -> instances/tags), apply_overrides previously
could not find the field because it only searched the raw (unexpanded)
template list.

The new _expand_and_find helper recursively loads nested template refs
in-place, allowing any named field at any nesting depth to be reached
from a single overrides: block in spec.yaml.

* Fix ruff formatting in template.py and test_template.py

Author: coignetp

datadog_checks_dev/changelog.d/23348.fixed

@@ -0,0 +1 @@
+Fix `apply_overrides` to support nested template references in spec.yaml overrides.

datadog_checks_dev/datadog_checks/dev/tooling/configuration/template.py

@@ -75,8 +75,45 @@ def load(self, template):
 
         return data
 
-    @staticmethod
-    def apply_overrides(template, overrides):
+    def _expand_and_find(self, items: list, name: str, visited: set | None = None) -> dict | None:
+        """Find a named item in a list, expanding nested template refs in-place as needed.
+
+        When the named item lives inside a nested ``- template:`` reference, that reference
+        is loaded and spliced into ``items`` in-place so that subsequent spec.py processing
+        sees the already-expanded data with any overrides applied.
+        """
+        if visited is None:
+            visited = set()
+
+        for item in items:
+            if isinstance(item, dict) and item.get('name') == name:
+                return item
+
+        for idx, item in enumerate(items):
+            if not isinstance(item, dict) or 'name' in item or 'template' not in item:
+                continue
+            tmpl_name = item['template']
+            if tmpl_name in visited:
+                continue
+            visited.add(tmpl_name)
+            try:
+                nested = self.load(tmpl_name)
+            except Exception:
+                continue
+
+            if isinstance(nested, dict):
+                if nested.get('name') == name:
+                    items[idx] = nested
+                    return nested
+            elif isinstance(nested, list):
+                found = self._expand_and_find(nested, name, visited)
+                if found is not None:
+                    items[idx : idx + 1] = nested
+                    return found
+
+        return None
+
+    def apply_overrides(self, template, overrides):
         errors = []
 
         for override, value in sorted(overrides.items()):
@@ -100,10 +137,9 @@ def apply_overrides(template, overrides):
                         )
                         break
                 elif isinstance(root, list):
-                    for item in root:
-                        if isinstance(item, dict) and item.get('name') == key:
-                            root = item
-                            break
+                    found = self._expand_and_find(root, key)
+                    if found is not None:
+                        root = found
                     else:
                         intermediate_error = (
                             f"Template override `{'.'.join(override_keys[:i])}` has no named mapping `{key}`"
@@ -123,10 +159,12 @@ def apply_overrides(template, overrides):
             if isinstance(root, dict):
                 root[final_key] = value
             elif isinstance(root, list):
-                for i, item in enumerate(root):
-                    if isinstance(item, dict) and item.get('name') == final_key:
-                        root[i] = value
-                        break
+                found = self._expand_and_find(root, final_key)
+                if found is not None:
+                    for i, item in enumerate(root):
+                        if item is found:
+                            root[i] = value
+                            break
                 else:
                     intermediate_error = 'Template override has no named mapping `{}`'.format(
                         '.'.join(override_keys) if override_keys else override

datadog_checks_dev/tests/tooling/configuration/test_template.py

@@ -230,3 +230,25 @@ def test_primitive_recurse(self):
 
         assert len(errors) == 1
         assert errors[0] == 'Template override `proxy.description` does not refer to a mapping'
+
+    def test_nested_template_override(self):
+        """Overrides should reach fields in nested template refs."""
+        templates = ConfigTemplates()
+
+        # instances/default → instances/all_integrations → instances/tags (tags field)
+        template = templates.load('instances/default')
+        errors = templates.apply_overrides(template, {'tags.display_priority': 5})
+        assert not errors
+
+        tags_item = next((item for item in template if isinstance(item, dict) and item.get('name') == 'tags'), None)
+        assert tags_item is not None
+        assert tags_item['display_priority'] == 5
+
+    def test_nested_template_override_not_found(self):
+        """An override for a name absent from nested templates still reports an error."""
+        templates = ConfigTemplates()
+
+        template = templates.load('instances/default')
+        errors = templates.apply_overrides(template, {'nonexistent_field.display_priority': 5})
+
+        assert len(errors) == 1