Detect stale and unreleased entries in Agent release requirements (DataDog#23813)

* Validate stale Agent release requirements

* Skip unreleased integrations in Agent release output

* Tighten unreleased-integrations parsing and broaden tests

- Validate that by-agent-version-range keys contain the '..' separator and raise a clear ValueError naming the offending key.
- Drop the redundant else branch in exclude_unreleased_integrations and move the historical folder-name comment back next to normalize_catalog.
- mkdir(exist_ok=True) in the write_repo_config helper for consistency with neighbours.
- Parametrize test_agent_version_in_range_is_inclusive (now covers below/above bounds and the malformed-range error path).
- Add a direct unit test on get_unreleased_integrations that exercises by-integration and by-agent-version-range together.
- Add a clean-pass test for validate agent-reqs and pull the set_root teardown into an isolated_root yield fixture.
- Add changelog entries for ddev and datadog_checks_dev.

* Surface malformed version ranges as a clean ddev abort

- Drop the redundant empty parent table header in .ddev/config.toml; the two sub-tables imply it.
- Catch ValueError from agent_version_in_range at every command entry point that triggers the lookup (integrations, changelog, integrations_changelog) and surface it via app.abort so config authors get a clean message instead of a Python traceback.
- Document that exclude_unreleased_integrations accepts both raw and folder-normalized catalog keys.

* Scope stale-entry detection to whole-file invocations

When the user runs `ddev validate agent-reqs <check>`, only the requested check should be validated; previously the new stale-entry detection still scanned the whole requirements file and surfaced unrelated stale packages, defeating per-check pre-commit usage.

Author: AAraKKe

.ddev/config.toml

@@ -240,6 +240,24 @@ trace-captures = false
 ## Just in case __pycache__ is present in the root of the repo
 __pycache__ = false
 
+# Integrations that were pinned in requirements-agent-release.txt but were not shipped
+# in the listed Agent releases. Agent release generation uses these entries to skip
+# false positives when building AGENT_INTEGRATIONS.md and Agent changelog data.
+# Use by-integration for one-off skips:
+#   integration_name = ["7.78.0", "7.79.0"]
+# Use by-agent-version-range for inclusive Agent version ranges:
+#   "7.74.0..7.78.0" = ["datadog-first-integration", "datadog-second-integration"]
+[overrides.release.agent.unreleased-integrations.by-integration]
+
+[overrides.release.agent.unreleased-integrations.by-agent-version-range]
+"7.74.0..7.78.0" = [
+  "datadog-control-m",
+  "datadog-krakend",
+  "datadog-lustre",
+  "datadog-n8n",
+  "datadog-prefect",
+]
+
 # Explicitely add the platforms supported by an integration for those where the manifest has been
 # removed.
 # This is a temporary fix while we implement a metadata.json file that we can add to each integration

datadog_checks_dev/changelog.d/23813.added

@@ -0,0 +1 @@
+Fail `ddev validate agent-reqs` when `requirements-agent-release.txt` pins a `datadog-*` package whose integration folder is no longer present in the repo.

datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/agent_reqs.py

@@ -13,9 +13,14 @@
     echo_warning,
 )
 from datadog_checks.dev.tooling.constants import AGENT_V5_ONLY, NOT_CHECKS, get_agent_release_requirements
-from datadog_checks.dev.tooling.release import get_package_name
+from datadog_checks.dev.tooling.release import get_folder_name, get_package_name
 from datadog_checks.dev.tooling.testing import process_checks_option
-from datadog_checks.dev.tooling.utils import complete_valid_checks, get_version_string, parse_agent_req_file
+from datadog_checks.dev.tooling.utils import (
+    complete_valid_checks,
+    get_valid_checks,
+    get_version_string,
+    parse_agent_req_file,
+)
 from datadog_checks.dev.utils import read_file
 
 
@@ -63,6 +68,33 @@ def agent_reqs(check):
     if unreleased_checks:
         joined_checks = ', '.join(unreleased_checks)
         echo_warning(f"{len(unreleased_checks)} unreleased checks: {joined_checks}")
+    if check is None or check.lower() == 'all':
+        stale_released_checks = find_stale_released_checks(agent_reqs_content)
+        if stale_released_checks:
+            failed_checks += len(stale_released_checks)
+            for package_name in stale_released_checks:
+                folder_name = get_folder_name(package_name)
+                message = (
+                    f"{package_name} is pinned in requirements-agent-release.txt "
+                    f"but `{folder_name}` is not present in the repo"
+                )
+                echo_failure(message)
+                annotate_error(release_requirements_file, message)
     if failed_checks:
         echo_failure(f"{failed_checks} checks out of sync")
         abort()
+
+
+def find_stale_released_checks(agent_reqs_content: dict[str, str]) -> list[str]:
+    """Return pinned Agent packages that no longer match a repo check."""
+    expected_packages = {
+        get_package_name(check_name)
+        for check_name in get_valid_checks()
+        if check_name not in AGENT_V5_ONLY | NOT_CHECKS
+    }
+
+    return sorted(
+        package_name
+        for package_name in agent_reqs_content
+        if package_name.startswith('datadog-') and package_name not in expected_packages
+    )

datadog_checks_dev/tests/tooling/commands/validate/test_agent_reqs.py

@@ -0,0 +1,66 @@
+# (C) Datadog, Inc. 2026-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+import os
+
+import pytest
+from click.testing import CliRunner
+
+from datadog_checks.dev.tooling.commands.validate.agent_reqs import agent_reqs
+from datadog_checks.dev.tooling.constants import get_root, set_root
+
+
+@pytest.fixture
+def isolated_root():
+    runner = CliRunner()
+    previous_root = get_root()
+    with runner.isolated_filesystem():
+        set_root(os.getcwd())
+        try:
+            yield runner
+        finally:
+            set_root(previous_root)
+
+
+def test_validate_agent_reqs_fails_on_stale_release_entry(isolated_root):
+    write_check('foo', '1.0.0')
+    with open('requirements-agent-release.txt', 'w', encoding='utf-8') as f:
+        f.write('# DO NOT PASS THIS TO PIP DIRECTLY\ndatadog-foo==1.0.0\ndatadog-snowflake==7.13.0\n')
+
+    result = isolated_root.invoke(agent_reqs)
+
+    assert result.exit_code == 1
+    assert (
+        'datadog-snowflake is pinned in requirements-agent-release.txt but `snowflake` is not present in the repo'
+    ) in result.output
+    assert 'datadog-foo is pinned' not in result.output
+
+
+def test_validate_agent_reqs_passes_when_every_entry_has_a_folder(isolated_root):
+    write_check('foo', '1.0.0')
+    with open('requirements-agent-release.txt', 'w', encoding='utf-8') as f:
+        f.write('# DO NOT PASS THIS TO PIP DIRECTLY\ndatadog-foo==1.0.0\n')
+
+    result = isolated_root.invoke(agent_reqs)
+
+    assert result.exit_code == 0
+    assert 'pinned in requirements-agent-release.txt' not in result.output
+
+
+def test_validate_agent_reqs_does_not_report_stale_entries_when_scoped_to_a_check(isolated_root):
+    write_check('foo', '1.0.0')
+    with open('requirements-agent-release.txt', 'w', encoding='utf-8') as f:
+        f.write('# DO NOT PASS THIS TO PIP DIRECTLY\ndatadog-foo==1.0.0\ndatadog-snowflake==7.13.0\n')
+
+    result = isolated_root.invoke(agent_reqs, ['foo'])
+
+    assert result.exit_code == 0
+    assert 'datadog-snowflake is pinned' not in result.output
+
+
+def write_check(name: str, version: str) -> None:
+    """Create the minimum check structure needed by agent-reqs."""
+    check_package = os.path.join(name, 'datadog_checks', name)
+    os.makedirs(check_package)
+    with open(os.path.join(check_package, '__about__.py'), 'w', encoding='utf-8') as f:
+        f.write(f'__version__ = "{version}"\n')

ddev/changelog.d/23813.added

@@ -0,0 +1 @@
+Skip integrations pinned in Agent release requirements but not actually shipped in a given Agent release, configurable under `[overrides.release.agent.unreleased-integrations]` in `.ddev/config.toml`.

ddev/src/ddev/cli/release/agent/changelog.py

@@ -58,7 +58,10 @@ def changelog(app: Application, since: str, to: str, write: bool, force: bool):
 
     app.repo.git.fetch_tags()
 
-    changes_per_agent = get_changes_per_agent(app.repo, since, to)
+    try:
+        changes_per_agent = get_changes_per_agent(app.repo, since, to)
+    except ValueError as exc:
+        app.abort(str(exc))
 
     # store the changelog in memory
     changelog_contents = StringIO()

ddev/src/ddev/cli/release/agent/common.py

@@ -11,6 +11,7 @@
     AgentChangelog = dict[str, dict[str, tuple[str, bool, bool]]]
 
 DATADOG_PACKAGE_PREFIX = 'datadog-'
+UNRELEASED_INTEGRATIONS_CONFIG = '/overrides/release/agent/unreleased-integrations'
 
 
 def get_agent_tags(repo: Repository, since: str, to: str) -> list[str]:
@@ -73,11 +74,8 @@ def get_changes_per_agent(repo: Repository, since: str, to: str) -> AgentChangel
         file_contents = repo.git.show_file(req_file_name, agent_tags[i])
         catalog_prev = parse_agent_req_file(file_contents)
 
-        # at some point in the git history, the requirements file erroneously
-        # contained the folder name instead of the package name for each check,
-        # let's be resilient by normalizing all entries to be folder names
-        catalog_now = normalize_catalog(catalog_now)
-        catalog_prev = normalize_catalog(catalog_prev)
+        catalog_now = exclude_unreleased_integrations(repo, normalize_catalog(catalog_now), current_tag)
+        catalog_prev = exclude_unreleased_integrations(repo, normalize_catalog(catalog_prev), agent_tags[i])
 
         changes_per_agent[current_tag] = {}
 
@@ -94,10 +92,56 @@ def get_changes_per_agent(repo: Repository, since: str, to: str) -> AgentChangel
     return changes_per_agent
 
 
+# at some point in the git history, the requirements file erroneously
+# contained the folder name instead of the package name for each check,
+# let's be resilient by normalizing all entries to be folder names
 def normalize_catalog(catalog: dict[str, str]) -> dict[str, str]:
     return {normalize_package_name(k): v for k, v in catalog.items()}
 
 
+def exclude_unreleased_integrations(repo: Repository, catalog: dict[str, str], agent_version: str) -> dict[str, str]:
+    """Filter integrations listed as unreleased for ``agent_version``; catalog keys may be raw or folder-normalized."""
+    skipped_integrations = get_unreleased_integrations(repo, agent_version)
+    if not skipped_integrations:
+        return catalog
+    return {
+        name: version for name, version in catalog.items() if normalize_package_name(name) not in skipped_integrations
+    }
+
+
+def get_unreleased_integrations(repo: Repository, agent_version: str) -> set[str]:
+    unreleased_integrations = repo.config.get(UNRELEASED_INTEGRATIONS_CONFIG, default={})
+    by_integration = unreleased_integrations.get('by-integration', {})
+    by_agent_version_range = unreleased_integrations.get('by-agent-version-range', {})
+
+    skipped_integrations = {
+        normalize_package_name(name) for name, versions in by_integration.items() if agent_version in versions
+    }
+    for version_range, integration_names in by_agent_version_range.items():
+        if agent_version_in_range(agent_version, version_range):
+            skipped_integrations.update(normalize_package_name(name) for name in integration_names)
+
+    return skipped_integrations
+
+
+def agent_version_in_range(agent_version: str, version_range: str) -> bool:
+    from packaging.version import parse as parse_version
+
+    parts = version_range.split('..', 1)
+    if len(parts) != 2:
+        raise ValueError(
+            f"Invalid version range {version_range!r} in "
+            f"{UNRELEASED_INTEGRATIONS_CONFIG}/by-agent-version-range; "
+            "expected format: 'START..END'"
+        )
+    start, end = parts
+    version = parse_version(agent_version)
+    start_version = parse_version(start)
+    end_version = parse_version(end)
+
+    return start_version <= version <= end_version
+
+
 def normalize_package_name(name: str) -> str:
     """
     Given a Python package name for a check, return the corresponding folder

ddev/src/ddev/cli/release/agent/integrations.py

@@ -29,7 +29,7 @@ def integrations(app: Application, since: str, to: str, write: bool, force: bool
     tool will generate the list for every Agent since version 6.3.0
     (before that point we don't have enough information to build the log).
     """
-    from ddev.cli.release.agent.common import get_agent_tags, parse_agent_req_file
+    from ddev.cli.release.agent.common import exclude_unreleased_integrations, get_agent_tags, parse_agent_req_file
 
     agent_tags = get_agent_tags(app.repo, since, to)
     # get the list of integrations shipped with the agent from the requirements file
@@ -40,7 +40,11 @@ def integrations(app: Application, since: str, to: str, write: bool, force: bool
         integrations_contents.write(f'## Datadog Agent version {tag}\n\n')
         # Requirements for current tag
         file_contents = app.repo.git.show_file(req_file_name, tag)
-        for name, ver in parse_agent_req_file(file_contents).items():
+        try:
+            catalog = exclude_unreleased_integrations(app.repo, parse_agent_req_file(file_contents), tag)
+        except ValueError as exc:
+            app.abort(str(exc))
+        for name, ver in catalog.items():
             integrations_contents.write(f'* {name}: {ver}\n')
         integrations_contents.write('\n')
 

ddev/src/ddev/cli/release/agent/integrations_changelog.py

@@ -39,7 +39,10 @@ def integrations_changelog(app: Application, integrations: tuple[str], since: st
     if not integrations:
         integrations = [integration.name for integration in app.repo.integrations.iter_all('all')]
 
-    changes_per_agent = get_changes_per_agent(app.repo, since, to)
+    try:
+        changes_per_agent = get_changes_per_agent(app.repo, since, to)
+    except ValueError as exc:
+        app.abort(str(exc))
 
     integrations_versions: dict[str, dict[str, str]] = defaultdict(dict)
     for agent_version, version_changes in changes_per_agent.items():

ddev/tests/cli/release/agent/conftest.py

@@ -1,9 +1,12 @@
 # (C) Datadog, Inc. 2023-present
 # All rights reserved
 # Licensed under a 3-clause BSD style license (see LICENSE)
+from collections.abc import Callable
+
 import pytest
 
 from ddev.repo.core import Repository
+from ddev.utils.fs import Path
 
 
 @pytest.fixture
@@ -55,6 +58,16 @@ def commit(msg):
     yield repo
 
 
+@pytest.fixture
+def write_repo_config() -> Callable[[Path, str], None]:
+    def write_config(repo_path: Path, contents: str) -> None:
+        config_dir = repo_path / '.ddev'
+        config_dir.mkdir(exist_ok=True)
+        (config_dir / 'config.toml').write_text(contents)
+
+    return write_config
+
+
 def write_agent_requirements(repo_path, requirements):
     with open(repo_path / 'requirements-agent-release.txt', 'w') as req_file:
         req_file.write('\n'.join(requirements))