Allow release tagging before agent branch exists (DataDog#23711)

Author: Kyle-Neale

.github/workflows/update-build-agent-yaml.yml

@@ -23,23 +23,48 @@ jobs:
         with:
           ref: ${{ inputs.branch }}
 
-      - name: Verify preconditions
+      - name: Update build_agent.yaml
         run: |-
-          if ! grep -qE '^\s+branch:\s+main\s*$' .gitlab/build_agent.yaml; then
-            echo "build_agent.yaml already points to the release branch — nothing to do."
+          python3 - <<'PY'
+          import os
+          import re
+          from pathlib import Path
+
+          path = Path(".gitlab/build_agent.yaml")
+          content = path.read_text()
+          template_regex = re.compile(r"^\.build-agent-tpl:\n(?:[^\S\n].*(?:\n|$))*", re.MULTILINE)
+          branch_regex = re.compile(r"^(\s+branch:\s+)main([^\S\n]*)$", re.MULTILINE)
+
+          template_match = template_regex.search(content)
+          if template_match is None:
+              print("build_agent.yaml does not contain `.build-agent-tpl` — nothing to update.")
+              raise SystemExit(0)
+
+          template = template_match.group(0)
+          matches = list(branch_regex.finditer(template))
+          if not matches:
+              print("`.build-agent-tpl` does not contain a branch entry pointing to main — nothing to update.")
+              raise SystemExit(0)
+          if len(matches) != 1:
+              print(f"::error::Expected exactly one `.build-agent-tpl` branch pointing to main; found {len(matches)}.")
+              raise SystemExit(1)
+
+          def replacement(match):
+              return f"{match.group(1)}{os.environ['BRANCH']}{match.group(2)}"
+
+          updated_template, replacement_count = branch_regex.subn(replacement, template, count=1)
+          assert replacement_count == 1
+          path.write_text(content[: template_match.start()] + updated_template + content[template_match.end() :])
+          PY
+
+          if git diff --quiet -- .gitlab/build_agent.yaml; then
             exit 0
           fi
           if ! git ls-remote --exit-code --heads https://github.com/DataDog/datadog-agent.git "$BRANCH" >/dev/null 2>&1; then
-            echo "::error::Agent branch '$BRANCH' does not exist in DataDog/datadog-agent yet."
-            exit 1
+            echo "::warning::Agent branch '$BRANCH' does not exist in DataDog/datadog-agent yet. Creating the PR anyway."
           fi
           echo "needs_update=true" >> "$GITHUB_ENV"
 
-      - name: Update build_agent.yaml
-        if: env.needs_update == 'true'
-        run: |-
-          sed -i "s/^    branch: main$/    branch: $BRANCH/" .gitlab/build_agent.yaml
-
       - name: Get token via dd-octo-sts
         if: env.needs_update == 'true'
         uses: DataDog/dd-octo-sts-action@96a25462dbcb10ebf0bfd6e2ccc917d2ab235b9a # v1.0.4

ddev/changelog.d/23711.fixed

@@ -0,0 +1 @@
+Allow release branch tagging to continue before the matching Agent branch exists.

ddev/src/ddev/cli/release/branch/create.py

@@ -17,8 +17,12 @@
 
 BRANCH_NAME_PATTERN = r"^\d+\.\d+\.x$"
 BRANCH_NAME_REGEX = re.compile(BRANCH_NAME_PATTERN)
+BUILD_AGENT_YAML_PATH = '.gitlab/build_agent.yaml'
+BUILD_AGENT_TEMPLATE_PATTERN = r'^\.build-agent-tpl:\n(?:[^\S\n].*(?:\n|$))*'
+BUILD_AGENT_MAIN_BRANCH_PATTERN = r'^(\s+branch:\s+)main([^\S\n]*)$'
+BUILD_AGENT_TEMPLATE_REGEX = re.compile(BUILD_AGENT_TEMPLATE_PATTERN, re.MULTILINE)
+BUILD_AGENT_MAIN_BRANCH_REGEX = re.compile(BUILD_AGENT_MAIN_BRANCH_PATTERN, re.MULTILINE)
 GITHUB_LABEL_COLOR = '5319e7'
-DATADOG_AGENT_REPO_URL = 'https://github.com/DataDog/datadog-agent.git'
 
 
 @click.command
@@ -178,58 +182,59 @@ def suggest_next_branch(app: Application) -> str:
 
 
 def ensure_build_agent_yaml_updated(app: Application, branch_name: str) -> bool:
-    """
-    Ensure build_agent.yaml points to the correct agent branch for release builds.
-
-    This function:
-    1. Checks if the file still points to 'main' (needs update)
-    2. Checks if the agent branch exists in datadog-agent repository
-    3. Updates the file if both conditions are met
-
-    Args:
-        branch_name: The release branch name (e.g., '7.45.x')
-
-    Returns:
-        True if the file was updated, False otherwise.
-    """
+    """Update build_agent.yaml to point to the release branch when it still targets main."""
     from ddev.utils.fs import Path
 
-    build_agent_yaml = Path('.gitlab/build_agent.yaml')
+    build_agent_yaml = Path(BUILD_AGENT_YAML_PATH)
 
     if not build_agent_yaml.exists():
         app.display_warning(f'Warning: {build_agent_yaml} not found')
         return False
 
-    # Read the current content
     with open(build_agent_yaml, 'r') as f:
         content = f.read()
 
-    # Check if file still points to main (needs update)
-    old_pattern = r'(\s+branch:\s+)main'
-    if not re.search(old_pattern, content):
-        # Already updated to a release branch, nothing to do
+    matches = find_build_agent_template_main_branch_matches(content)
+    if not matches:
         return False
-
-    # Check if the agent branch exists in datadog-agent repository using git ls-remote
-    app.display_waiting(f'Checking if branch `{branch_name}` exists in datadog-agent...')
-    ls_remote_output = app.repo.git.capture('ls-remote', '--heads', DATADOG_AGENT_REPO_URL, branch_name)
-    if not ls_remote_output.strip():
-        app.display_warning(
-            f"Agent branch `{branch_name}` does not exist yet in datadog-agent. "
-            f"Keeping build_agent.yaml pointing to 'main'. "
-            f"The `update-build-agent-yaml` workflow will create a PR to update the file "
-            f"once the agent branch is created."
+    if len(matches) > 1:
+        app.abort(
+            f'Expected exactly one `.build-agent-tpl` branch pointing to `main` in `{BUILD_AGENT_YAML_PATH}`; '
+            f'found {len(matches)}.'
         )
         return False
 
-    # Agent branch exists, update the file
-    def replacement(match):
-        return match.group(1) + branch_name
-
-    updated_content = re.sub(old_pattern, replacement, content)
+    updated_content, replacement_count = replace_build_agent_template_main_branch(content, branch_name)
+    assert replacement_count == 1
 
     with open(build_agent_yaml, 'w') as f:
         f.write(updated_content)
 
     app.display_success(f'Updated build_agent.yaml file to use Agent branch: {branch_name}')
     return True
+
+
+def find_build_agent_template_main_branch_matches(content: str) -> list[re.Match[str]]:
+    template_match = BUILD_AGENT_TEMPLATE_REGEX.search(content)
+    if template_match is None:
+        return []
+
+    return list(BUILD_AGENT_MAIN_BRANCH_REGEX.finditer(template_match.group(0)))
+
+
+def replace_build_agent_template_main_branch(content: str, branch_name: str) -> tuple[str, int]:
+    template_match = BUILD_AGENT_TEMPLATE_REGEX.search(content)
+    if template_match is None:
+        return content, 0
+
+    def replacement(match: re.Match[str]) -> str:
+        return f'{match.group(1)}{branch_name}{match.group(2)}'
+
+    updated_template, replacement_count = BUILD_AGENT_MAIN_BRANCH_REGEX.subn(
+        replacement, template_match.group(0), count=1
+    )
+    if replacement_count == 0:
+        return content, 0
+
+    updated_content = content[: template_match.start()] + updated_template + content[template_match.end() :]
+    return updated_content, replacement_count

ddev/src/ddev/cli/release/branch/tag.py

@@ -1,10 +1,21 @@
+from __future__ import annotations
+
 import logging
 import re
+from typing import TYPE_CHECKING
 
 import click
+from httpx import HTTPStatusError
 from packaging.version import Version
 
-from .create import BRANCH_NAME_REGEX
+from .create import BRANCH_NAME_REGEX, BUILD_AGENT_YAML_PATH, find_build_agent_template_main_branch_matches
+
+if TYPE_CHECKING:
+    from ddev.cli.application import Application
+
+UPDATE_BUILD_AGENT_YAML_WORKFLOW = 'update-build-agent-yaml.yml'
+# The dd-octo-sts policy grants PR-writing credentials only to this workflow on master.
+UPDATE_BUILD_AGENT_YAML_WORKFLOW_REF = 'master'
 
 
 @click.command
@@ -36,12 +47,14 @@ def tag(app, final: bool, skip_open_pr_check: bool):
     click.echo(app.repo.git.pull(branch_name))
     click.echo(app.repo.git.fetch_tags())
 
-    if _build_agent_yaml_points_to_main():
-        app.abort(
+    build_agent_yaml_needs_update = _build_agent_yaml_points_to_main()
+    # Recovery path for release branches cut before build_agent.yaml was updated.
+    if build_agent_yaml_needs_update:
+        app.display_warning(
             "`.gitlab/build_agent.yaml` still points to `main`.\n"
-            "The agent branch may not exist yet in datadog-agent, or the update PR hasn't been merged.\n"
-            f"To trigger the workflow manually: gh workflow run update-build-agent-yaml.yml -f branch={branch_name}\n"
-            "Once the PR is merged, re-run this command."
+            "The update PR may not have been created or merged yet.\n"
+            "Will trigger the workflow after the tag is pushed.\n"
+            "Tagging will continue."
         )
 
     major_minor_version = branch_name.replace('.x', '')
@@ -105,13 +118,33 @@ def tag(app, final: bool, skip_open_pr_check: bool):
         app.abort('Did not get confirmation, aborting. Did not create or push the tag.')
     click.echo(app.repo.git.tag(new_tag, message=new_tag))
     click.echo(app.repo.git.push(new_tag))
+    if build_agent_yaml_needs_update:
+        _trigger_build_agent_yaml_update_workflow(app, branch_name)
 
 
 def _build_agent_yaml_points_to_main() -> bool:
     from ddev.utils.fs import Path
 
-    path = Path('.gitlab/build_agent.yaml')
-    return path.exists() and bool(re.search(r'\s+branch:\s+main$', path.read_text(), re.MULTILINE))
+    path = Path(BUILD_AGENT_YAML_PATH)
+    return path.exists() and bool(find_build_agent_template_main_branch_matches(path.read_text()))
+
+
+def _trigger_build_agent_yaml_update_workflow(app: Application, branch_name: str) -> None:
+    try:
+        app.github.dispatch_workflow(
+            UPDATE_BUILD_AGENT_YAML_WORKFLOW,
+            UPDATE_BUILD_AGENT_YAML_WORKFLOW_REF,
+            {'branch': branch_name},
+        )
+    except HTTPStatusError as e:
+        app.display_warning(
+            f'Warning: unable to trigger `{UPDATE_BUILD_AGENT_YAML_WORKFLOW}`: {e}\n'
+            f'To trigger it manually: gh workflow run {UPDATE_BUILD_AGENT_YAML_WORKFLOW} -f branch={branch_name}'
+        )
+    else:
+        app.display_success(
+            f'Dispatched `{UPDATE_BUILD_AGENT_YAML_WORKFLOW}`; check the workflow run for PR creation status.'
+        )
 
 
 def _extract_patch_and_rc(version_tags):

ddev/tests/cli/release/branch/test_create.py

@@ -32,8 +32,8 @@ def test_create_invalid_branch_name(ddev, name, mocker):
 @pytest.mark.parametrize(
     'yaml_updated',
     [
-        pytest.param(True, id='agent_branch_exists'),
-        pytest.param(False, id='agent_branch_not_exists'),
+        pytest.param(True, id='build_agent_yaml_updated'),
+        pytest.param(False, id='build_agent_yaml_unchanged'),
     ],
 )
 def test_create_branch(ddev, mocker, yaml_updated):
@@ -58,7 +58,7 @@ def test_create_branch(ddev, mocker, yaml_updated):
     run_mock.assert_any_call('checkout', '-B', '5.5.x')
     run_mock.assert_any_call('push', 'origin', '5.5.x')
 
-    # yaml commit only happens when agent branch exists
+    # yaml commit only happens when build_agent.yaml was updated
     yaml_commit = call('add', '.gitlab/build_agent.yaml')
     assert (yaml_commit in run_mock.call_args_list) is yaml_updated
 
@@ -78,45 +78,73 @@ def test_create_branch(ddev, mocker, yaml_updated):
     assert run_mock.call_args_list[-1] == call('checkout', 'master')
 
 
-@pytest.mark.parametrize(
-    'ls_remote_output,expected_result,file_should_change',
-    [
-        pytest.param('abc123\trefs/heads/7.99.x\n', True, True, id='branch_exists'),
-        pytest.param('', False, False, id='branch_not_exists'),
-    ],
-)
-def test_ensure_build_agent_yaml_updated(mocker, tmp_path, ls_remote_output, expected_result, file_should_change):
-    """Test ensure_build_agent_yaml_updated with different branch existence scenarios."""
+def test_ensure_build_agent_yaml_updated(mocker, tmp_path):
     build_agent_path = Path(tmp_path / '.gitlab' / 'build_agent.yaml')
     build_agent_path.parent.ensure_dir_exists()
     build_agent_path.write_text('.build-agent-tpl:\n  trigger:\n    branch: main\n')
 
     app_mock = mocker.MagicMock()
-    app_mock.repo.git.capture.return_value = ls_remote_output
 
     with Path(tmp_path).as_cwd():
         result = ensure_build_agent_yaml_updated(app_mock, '7.99.x')
 
-    assert result is expected_result
-    content = build_agent_path.read_text()
-    if file_should_change:
-        assert 'branch: 7.99.x' in content
-    else:
-        assert 'branch: main' in content
+    assert result is True
+    assert 'branch: 7.99.x' in build_agent_path.read_text()
+    app_mock.repo.git.capture.assert_not_called()
+
+
+def test_ensure_build_agent_yaml_updated_ignores_unrelated_main_branch(mocker, tmp_path):
+    build_agent_path = Path(tmp_path / '.gitlab' / 'build_agent.yaml')
+    build_agent_path.parent.ensure_dir_exists()
+    content = '.build-agent-tpl:\n  trigger:\n    branch: main\nunrelated-job:\n  trigger:\n    branch: main\n'
+    build_agent_path.write_text(content)
+
+    app_mock = mocker.MagicMock()
+
+    with Path(tmp_path).as_cwd():
+        result = ensure_build_agent_yaml_updated(app_mock, '7.99.x')
+
+    assert result is True
+    assert build_agent_path.read_text() == (
+        '.build-agent-tpl:\n  trigger:\n    branch: 7.99.x\nunrelated-job:\n  trigger:\n    branch: main\n'
+    )
+    app_mock.abort.assert_not_called()
+
+
+def test_ensure_build_agent_yaml_updated_aborts_on_multiple_template_main_branches(mocker, tmp_path):
+    build_agent_path = Path(tmp_path / '.gitlab' / 'build_agent.yaml')
+    build_agent_path.parent.ensure_dir_exists()
+    content = '.build-agent-tpl:\n  trigger:\n    branch: main\n    branch: main\n'
+    build_agent_path.write_text(content)
+
+    app_mock = mocker.MagicMock()
+    app_mock.abort.side_effect = RuntimeError('abort')
+
+    with Path(tmp_path).as_cwd(), pytest.raises(RuntimeError, match='abort'):
+        ensure_build_agent_yaml_updated(app_mock, '7.99.x')
+
+    assert build_agent_path.read_text() == content
+    app_mock.abort.assert_called_once_with(
+        'Expected exactly one `.build-agent-tpl` branch pointing to `main` in `.gitlab/build_agent.yaml`; found 2.'
+    )
 
 
 def test_ensure_build_agent_yaml_updated_already_on_release_branch(mocker, tmp_path):
     """Test early return when file already points to a release branch."""
     build_agent_path = Path(tmp_path / '.gitlab' / 'build_agent.yaml')
     build_agent_path.parent.ensure_dir_exists()
-    build_agent_path.write_text('.build-agent-tpl:\n  trigger:\n    branch: 7.98.x\n')
+    build_agent_path.write_text(
+        '.build-agent-tpl:\n  trigger:\n    branch: 7.98.x\nunrelated-job:\n  trigger:\n    branch: main\n'
+    )
 
     app_mock = mocker.MagicMock()
 
     with Path(tmp_path).as_cwd():
         result = ensure_build_agent_yaml_updated(app_mock, '7.99.x')
 
     assert result is False
+    assert 'branch: 7.98.x' in build_agent_path.read_text()
+    assert 'branch: main' in build_agent_path.read_text()
     app_mock.repo.git.capture.assert_not_called()