Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ddev/changelog.d/23646.fixed
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Retry agent check invocations on transient failures to address SNMP E2E flake from autodiscovery reload races.
29 changes: 26 additions & 3 deletions ddev/src/ddev/cli/env/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,26 @@

if TYPE_CHECKING:
from ddev.cli.application import Application
from ddev.e2e.agent.interface import AgentInterface


def _invoke_check_with_retry(agent: AgentInterface, args: list[str], *, retries: int = 3, backoff: float = 0.5) -> None:
"""Invoke ``agent check`` with bounded retry to absorb transient autodiscovery-reload races."""
import subprocess
import time

for attempt in range(retries + 1):
try:
agent.invoke(args)
return
except subprocess.CalledProcessError:
if attempt >= retries:
raise
click.echo(
f'agent check failed (attempt {attempt + 1}/{retries + 1}), retrying in {backoff:.1f}s...',
err=True,
)
time.sleep(backoff)


@click.command(
Expand Down Expand Up @@ -54,7 +74,10 @@ def agent(app: Application, *, intg_name: str, environment: str, args: tuple[str

if config_file is None or not trigger_run:
try:
agent.invoke(full_args)
if trigger_run:
_invoke_check_with_retry(agent, full_args)
else:
agent.invoke(full_args)
except subprocess.CalledProcessError as e:
app.abort(code=e.returncode)

Expand All @@ -67,14 +90,14 @@ def agent(app: Application, *, intg_name: str, environment: str, args: tuple[str
if not env_data.config_file.is_file():
try:
env_data.write_config(config)
agent.invoke(full_args)
_invoke_check_with_retry(agent, full_args)
finally:
env_data.config_file.unlink()
else:
temp_config_file = env_data.config_file.parent / f'{env_data.config_file.name}.bak.example'
env_data.config_file.replace(temp_config_file)
try:
env_data.write_config(config)
agent.invoke(full_args)
_invoke_check_with_retry(agent, full_args)
finally:
temp_config_file.replace(env_data.config_file)
Loading