Fix false-positive discovery-candidate stability failures from log concatenation (DataDog#24536)

* Fix false-positive discovery-candidate stability failures from log concatenation

assert_all_discovery_candidates_stable diffed docker logs by concatenating
stdout and stderr before comparing against the previous snapshot. Any new
stdout content (e.g. an access log line) shifts where the unchanged stderr
tail sits in the concatenation, breaking the startswith-based diff and
replaying the container's entire historical stderr output as "new" on every
probe. This caused Kong's E2E discovery test to intermittently fail with
"Container logs matched 'error'" whenever a benign startup-time nginx/
migrations message already sat in stderr. Diff stdout and stderr
independently instead.

Environment: Datadog workspace

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

* Add changelog entry for DataDog#24536

Environment: Datadog workspace

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

* Trim internal helper docstring/comment to one line

AGENTS.md requires internal helpers to use concise one-line docstrings/comments;
the new _diff_logs docstring and the regression test comment were multi-line.

Environment: Datadog workspace

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>

Author: vitkyrka

datadog_checks_dev/changelog.d/24536.fixed

@@ -0,0 +1 @@
+Fix false-positive discovery-candidate stability failures caused by incorrectly diffing concatenated stdout/stderr container logs.

datadog_checks_dev/datadog_checks/dev/docker.py

@@ -72,7 +72,7 @@ def assert_all_discovery_candidates_stable(
     compose_service = compose_service or _get_default_compose_service()
     container_id = _get_compose_container_id(compose_file, compose_service, project_name=project_name)
     initial_state = _inspect_container(container_id)
-    previous_logs = _get_container_logs(container_id)
+    previous_stdout, previous_stderr = _get_container_logs(container_id)
 
     ports = tuple(SimpleNamespace(number=port, name='') for port in _get_container_ports(initial_state))
     service = SimpleNamespace(
@@ -100,12 +100,15 @@ def assert_all_discovery_candidates_stable(
         current_state = _inspect_container(current_container_id)
         _assert_container_stable(initial_state, current_state, index)
 
-        current_logs = _get_container_logs(current_container_id)
-        new_logs = current_logs[len(previous_logs) :] if current_logs.startswith(previous_logs) else current_logs
+        current_stdout, current_stderr = _get_container_logs(current_container_id)
+        # Diffed separately: stdout/stderr interleaving varies across calls, breaking a combined diff.
+        new_stdout = _diff_logs(previous_stdout, current_stdout)
+        new_stderr = _diff_logs(previous_stderr, current_stderr)
+        new_logs = new_stdout + new_stderr
         for line in new_logs.splitlines():
             logging.debug('New log line: %s', line)
         _assert_no_log_patterns(new_logs, log_patterns, index)
-        previous_logs = current_logs
+        previous_stdout, previous_stderr = current_stdout, current_stderr
 
 
 def _get_compose_container_id(
@@ -188,9 +191,14 @@ def _get_container_ports(inspect_data: Mapping[str, Any]) -> list[int]:
     return sorted(ports)
 
 
-def _get_container_logs(container_id: str) -> str:
+def _get_container_logs(container_id: str) -> tuple[str, str]:
     result = run_command(['docker', 'logs', container_id], capture=True)
-    return result.stdout + result.stderr
+    return result.stdout, result.stderr
+
+
+def _diff_logs(previous: str, current: str) -> str:
+    """Return the portion of `current` appended since `previous`."""
+    return current[len(previous) :] if current.startswith(previous) else current
 
 
 def _assert_container_stable(

datadog_checks_dev/tests/test_docker.py

@@ -394,3 +394,35 @@ def fake_run_command(command, **kwargs):
                 '/tmp/docker-compose.yml',
                 'service',
             )
+
+
+def test_assert_all_discovery_candidates_stable_ignores_old_stderr_when_new_stdout_arrives():
+    # Regression test: new stdout previously shifted unchanged stderr, replaying it as new.
+    class DiscoveryCheck:
+        @classmethod
+        def generate_configs(cls, service):
+            yield {'init_config': {}, 'instances': [{'url': f'http://{service.host}:8080/metrics'}]}
+
+    logs = iter(
+        [
+            _docker_result(stdout='', stderr='old startup error\n'),
+            _docker_result(stdout='new access log line\n', stderr='old startup error\n'),
+        ]
+    )
+
+    def fake_run_command(command, **kwargs):
+        if command[:2] == ['docker', 'compose']:
+            return _docker_result(stdout='container-id\n')
+        if command[:2] == ['docker', 'inspect']:
+            return _docker_result(stdout=json.dumps([_container_inspect()]))
+        if command[:2] == ['docker', 'logs']:
+            return next(logs)
+        raise AssertionError(f'Unexpected command: {command}')
+
+    with mock.patch('datadog_checks.dev.docker.run_command', side_effect=fake_run_command):
+        assert_all_discovery_candidates_stable(
+            mock.Mock(),
+            DiscoveryCheck,
+            '/tmp/docker-compose.yml',
+            'service',
+        )