Guard build_agent.yaml updates on Agent branch existence (DataDog#23987)

* Guard build_agent.yaml updates on Agent branch existence

Extract YAML helpers into a shared build_agent.py module so the inline
update in `ddev release branch create` and the recovery path through
the workflow share parsing logic. Gate both writers on the matching
DataDog/datadog-agent branch existing:

- `ensure_build_agent_yaml_updated` skips the rewrite (with a warning)
  when the upstream branch is missing, leaving `main` in place for the
  tag-time recovery path to handle later.
- The `update-build-agent-yaml.yml` workflow now hard-fails instead of
  warning when the upstream branch is missing, so a fire-and-forget
  dispatch from `ddev release branch tag` is visible in Actions.
- `bump_milestone` defensively restores build_agent.yaml from
  origin/master after checking out the milestone-bump branch so the
  release-branch edit cannot leak into the bump commit
  (see PR DataDog#23977 commit f71a89c).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* Add changelog entry

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Wrap warning message to satisfy ruff E501

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* Drop unused re-exports and __all__ from create.py

No external caller imports the two YAML-helper names from create.py;
they live in build_agent.py and tag.py imports them from there
directly. With those imports gone, __all__ has no purpose either.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* Import build agent helper from owner module

---------

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

Author: 3 people

.github/workflows/update-build-agent-yaml.yml

@@ -61,7 +61,8 @@ jobs:
             exit 0
           fi
           if ! git ls-remote --exit-code --heads https://github.com/DataDog/datadog-agent.git "$BRANCH" >/dev/null 2>&1; then
-            echo "::warning::Agent branch '$BRANCH' does not exist in DataDog/datadog-agent yet. Creating the PR anyway."
+            echo "::error::Agent branch '$BRANCH' does not exist in DataDog/datadog-agent. Refusing to open a PR pointing to a missing upstream branch. Re-dispatch this workflow once the upstream branch is cut."
+            exit 1
           fi
           echo "needs_update=true" >> "$GITHUB_ENV"
 

ddev/changelog.d/23987.fixed

@@ -0,0 +1 @@
+Gate `ddev release branch create` and `update-build-agent-yaml.yml` on the matching `DataDog/datadog-agent` branch existing so neither writer can produce a release-branch pointer to a missing upstream branch.

ddev/src/ddev/cli/release/branch/build_agent.py

@@ -0,0 +1,101 @@
+# (C) Datadog, Inc. 2024-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+from __future__ import annotations
+
+import re
+import subprocess
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ddev.cli.application import Application
+
+BUILD_AGENT_YAML_PATH = '.gitlab/build_agent.yaml'
+BUILD_AGENT_TEMPLATE_PATTERN = r'^\.build-agent-tpl:\n(?:[^\S\n].*(?:\n|$))*'
+BUILD_AGENT_MAIN_BRANCH_PATTERN = r'^(\s+branch:\s+)main([^\S\n]*)$'
+BUILD_AGENT_TEMPLATE_REGEX = re.compile(BUILD_AGENT_TEMPLATE_PATTERN, re.MULTILINE)
+BUILD_AGENT_MAIN_BRANCH_REGEX = re.compile(BUILD_AGENT_MAIN_BRANCH_PATTERN, re.MULTILINE)
+
+DATADOG_AGENT_REPO_URL = 'https://github.com/DataDog/datadog-agent.git'
+
+
+def agent_branch_exists(branch_name: str) -> bool:
+    """Return ``True`` if ``branch_name`` exists in ``DataDog/datadog-agent``."""
+    result = subprocess.run(
+        ['git', 'ls-remote', '--exit-code', '--heads', DATADOG_AGENT_REPO_URL, branch_name],
+        capture_output=True,
+        check=False,
+    )
+    return result.returncode == 0
+
+
+def find_build_agent_template_main_branch_matches(content: str) -> list[re.Match[str]]:
+    template_match = BUILD_AGENT_TEMPLATE_REGEX.search(content)
+    if template_match is None:
+        return []
+    return list(BUILD_AGENT_MAIN_BRANCH_REGEX.finditer(template_match.group(0)))
+
+
+def replace_build_agent_template_main_branch(content: str, branch_name: str) -> tuple[str, int]:
+    template_match = BUILD_AGENT_TEMPLATE_REGEX.search(content)
+    if template_match is None:
+        return content, 0
+
+    def replacement(match: re.Match[str]) -> str:
+        return f'{match.group(1)}{branch_name}{match.group(2)}'
+
+    updated_template, replacement_count = BUILD_AGENT_MAIN_BRANCH_REGEX.subn(
+        replacement, template_match.group(0), count=1
+    )
+    if replacement_count == 0:
+        return content, 0
+
+    updated_content = content[: template_match.start()] + updated_template + content[template_match.end() :]
+    return updated_content, replacement_count
+
+
+def ensure_build_agent_yaml_updated(app: Application, branch_name: str) -> bool:
+    """Update build_agent.yaml to point to the release branch when it still targets main.
+
+    Returns False without modifying the file when the matching ``DataDog/datadog-agent``
+    branch does not exist yet, so callers can leave ``main`` in place for the recovery
+    path (``ddev release branch tag`` -> ``update-build-agent-yaml.yml``) to run later.
+    """
+    from ddev.utils.fs import Path
+
+    build_agent_yaml = Path(BUILD_AGENT_YAML_PATH)
+
+    if not build_agent_yaml.exists():
+        app.display_warning(f'Warning: {build_agent_yaml} not found')
+        return False
+
+    with open(build_agent_yaml, 'r') as f:
+        content = f.read()
+
+    matches = find_build_agent_template_main_branch_matches(content)
+    if not matches:
+        return False
+    if len(matches) > 1:
+        app.abort(
+            f'Expected exactly one `.build-agent-tpl` branch pointing to `main` in `{BUILD_AGENT_YAML_PATH}`; '
+            f'found {len(matches)}.'
+        )
+        return False
+
+    if not agent_branch_exists(branch_name):
+        app.display_warning(
+            f'Unable to verify that agent branch `{branch_name}` exists in `DataDog/datadog-agent`. '
+            f'Leaving `{BUILD_AGENT_YAML_PATH}` pointing to `main`. '
+            f'Re-dispatch `update-build-agent-yaml.yml` (or re-run `ddev release branch tag`) '
+            f'once the upstream branch exists.'
+        )
+        return False
+
+    updated_content, replacement_count = replace_build_agent_template_main_branch(content, branch_name)
+    assert replacement_count == 1
+
+    with open(build_agent_yaml, 'w') as f:
+        f.write(updated_content)
+
+    app.display_success(f'Updated build_agent.yaml file to use Agent branch: {branch_name}')
+    return True

ddev/src/ddev/cli/release/branch/create.py

@@ -12,16 +12,13 @@
 from httpx import HTTPError, HTTPStatusError
 from packaging.version import Version
 
+from .build_agent import BUILD_AGENT_YAML_PATH, ensure_build_agent_yaml_updated
+
 if TYPE_CHECKING:
     from ddev.cli.application import Application
 
 BRANCH_NAME_PATTERN = r"^\d+\.\d+\.x$"
 BRANCH_NAME_REGEX = re.compile(BRANCH_NAME_PATTERN)
-BUILD_AGENT_YAML_PATH = '.gitlab/build_agent.yaml'
-BUILD_AGENT_TEMPLATE_PATTERN = r'^\.build-agent-tpl:\n(?:[^\S\n].*(?:\n|$))*'
-BUILD_AGENT_MAIN_BRANCH_PATTERN = r'^(\s+branch:\s+)main([^\S\n]*)$'
-BUILD_AGENT_TEMPLATE_REGEX = re.compile(BUILD_AGENT_TEMPLATE_PATTERN, re.MULTILINE)
-BUILD_AGENT_MAIN_BRANCH_REGEX = re.compile(BUILD_AGENT_MAIN_BRANCH_PATTERN, re.MULTILINE)
 GITHUB_LABEL_COLOR = '5319e7'
 
 
@@ -108,6 +105,9 @@ def bump_milestone(app: Application, branch_name: str) -> None:
         app.display_waiting(f"Updating release.json with new milestone `{next_milestone}`...")
         app.repo.git.run('fetch', 'origin', 'master')
         app.repo.git.run('checkout', '-B', bump_branch, 'origin/master')
+        # Force-restore build_agent.yaml from origin/master so any prior in-process edit on the release
+        # branch cannot leak into the milestone-bump commit (root cause of the leak on PR #23977).
+        app.repo.git.run('checkout', 'origin/master', '--', BUILD_AGENT_YAML_PATH)
         update_release_json(app.repo.path / 'release.json', next_milestone)
         app.repo.git.run('add', 'release.json')
         app.repo.git.run('commit', '-m', f'Update current_milestone to {next_milestone}')
@@ -179,62 +179,3 @@ def suggest_next_branch(app: Application) -> str:
             return suggestion
 
     return f'{major}.{minors[-1] + 1}.x'
-
-
-def ensure_build_agent_yaml_updated(app: Application, branch_name: str) -> bool:
-    """Update build_agent.yaml to point to the release branch when it still targets main."""
-    from ddev.utils.fs import Path
-
-    build_agent_yaml = Path(BUILD_AGENT_YAML_PATH)
-
-    if not build_agent_yaml.exists():
-        app.display_warning(f'Warning: {build_agent_yaml} not found')
-        return False
-
-    with open(build_agent_yaml, 'r') as f:
-        content = f.read()
-
-    matches = find_build_agent_template_main_branch_matches(content)
-    if not matches:
-        return False
-    if len(matches) > 1:
-        app.abort(
-            f'Expected exactly one `.build-agent-tpl` branch pointing to `main` in `{BUILD_AGENT_YAML_PATH}`; '
-            f'found {len(matches)}.'
-        )
-        return False
-
-    updated_content, replacement_count = replace_build_agent_template_main_branch(content, branch_name)
-    assert replacement_count == 1
-
-    with open(build_agent_yaml, 'w') as f:
-        f.write(updated_content)
-
-    app.display_success(f'Updated build_agent.yaml file to use Agent branch: {branch_name}')
-    return True
-
-
-def find_build_agent_template_main_branch_matches(content: str) -> list[re.Match[str]]:
-    template_match = BUILD_AGENT_TEMPLATE_REGEX.search(content)
-    if template_match is None:
-        return []
-
-    return list(BUILD_AGENT_MAIN_BRANCH_REGEX.finditer(template_match.group(0)))
-
-
-def replace_build_agent_template_main_branch(content: str, branch_name: str) -> tuple[str, int]:
-    template_match = BUILD_AGENT_TEMPLATE_REGEX.search(content)
-    if template_match is None:
-        return content, 0
-
-    def replacement(match: re.Match[str]) -> str:
-        return f'{match.group(1)}{branch_name}{match.group(2)}'
-
-    updated_template, replacement_count = BUILD_AGENT_MAIN_BRANCH_REGEX.subn(
-        replacement, template_match.group(0), count=1
-    )
-    if replacement_count == 0:
-        return content, 0
-
-    updated_content = content[: template_match.start()] + updated_template + content[template_match.end() :]
-    return updated_content, replacement_count

ddev/src/ddev/cli/release/branch/tag.py

@@ -8,7 +8,8 @@
 from httpx import HTTPStatusError
 from packaging.version import Version
 
-from .create import BRANCH_NAME_REGEX, BUILD_AGENT_YAML_PATH, find_build_agent_template_main_branch_matches
+from .build_agent import BUILD_AGENT_YAML_PATH, find_build_agent_template_main_branch_matches
+from .create import BRANCH_NAME_REGEX
 
 if TYPE_CHECKING:
     from ddev.cli.application import Application

ddev/tests/cli/release/branch/test_create.py

@@ -8,7 +8,8 @@
 
 import pytest
 
-from ddev.cli.release.branch.create import compute_next_milestone, ensure_build_agent_yaml_updated, update_release_json
+from ddev.cli.release.branch.build_agent import ensure_build_agent_yaml_updated
+from ddev.cli.release.branch.create import compute_next_milestone, update_release_json
 from ddev.utils.fs import Path
 
 
@@ -70,6 +71,9 @@ def test_create_branch(ddev, mocker, yaml_updated):
     # Milestone bump workflow
     run_mock.assert_any_call('fetch', 'origin', 'master')
     run_mock.assert_any_call('checkout', '-B', 'release/bump-milestone-5.6.0', 'origin/master')
+    # Defensive restore prevents the build_agent.yaml change on the release branch
+    # from leaking into the milestone-bump commit (see PR #23977 leak).
+    run_mock.assert_any_call('checkout', 'origin/master', '--', '.gitlab/build_agent.yaml')
     run_mock.assert_any_call('add', 'release.json')
     run_mock.assert_any_call('commit', '-m', 'Update current_milestone to 5.6.0')
     run_mock.assert_any_call('push', 'origin', 'release/bump-milestone-5.6.0')
@@ -84,6 +88,7 @@ def test_ensure_build_agent_yaml_updated(mocker, tmp_path):
     build_agent_path.write_text('.build-agent-tpl:\n  trigger:\n    branch: main\n')
 
     app_mock = mocker.MagicMock()
+    mocker.patch('ddev.cli.release.branch.build_agent.agent_branch_exists', return_value=True)
 
     with Path(tmp_path).as_cwd():
         result = ensure_build_agent_yaml_updated(app_mock, '7.99.x')
@@ -100,6 +105,7 @@ def test_ensure_build_agent_yaml_updated_ignores_unrelated_main_branch(mocker, t
     build_agent_path.write_text(content)
 
     app_mock = mocker.MagicMock()
+    mocker.patch('ddev.cli.release.branch.build_agent.agent_branch_exists', return_value=True)
 
     with Path(tmp_path).as_cwd():
         result = ensure_build_agent_yaml_updated(app_mock, '7.99.x')
@@ -111,6 +117,27 @@ def test_ensure_build_agent_yaml_updated_ignores_unrelated_main_branch(mocker, t
     app_mock.abort.assert_not_called()
 
 
+def test_ensure_build_agent_yaml_updated_skips_when_agent_branch_missing(mocker, tmp_path):
+    """Leave `main` in place when the matching DataDog/datadog-agent branch does not exist yet."""
+    build_agent_path = Path(tmp_path / '.gitlab' / 'build_agent.yaml')
+    build_agent_path.parent.ensure_dir_exists()
+    original_content = '.build-agent-tpl:\n  trigger:\n    branch: main\n'
+    build_agent_path.write_text(original_content)
+
+    app_mock = mocker.MagicMock()
+    mocker.patch('ddev.cli.release.branch.build_agent.agent_branch_exists', return_value=False)
+
+    with Path(tmp_path).as_cwd():
+        result = ensure_build_agent_yaml_updated(app_mock, '7.99.x')
+
+    assert result is False
+    assert build_agent_path.read_text() == original_content
+    app_mock.display_warning.assert_called_once()
+    warning_message = app_mock.display_warning.call_args[0][0]
+    assert '7.99.x' in warning_message
+    assert 'datadog-agent' in warning_message
+
+
 def test_ensure_build_agent_yaml_updated_aborts_on_multiple_template_main_branches(mocker, tmp_path):
     build_agent_path = Path(tmp_path / '.gitlab' / 'build_agent.yaml')
     build_agent_path.parent.ensure_dir_exists()