From 7d373bfa46af083edff6a473614e36ea04368baf Mon Sep 17 00:00:00 2001 From: Julien Doutre <36448022+juliendoutre@users.noreply.github.com> Date: Mon, 25 Aug 2025 10:49:57 +0200 Subject: [PATCH 1/4] Pin GitHub Actions (#21093) --- .github/workflows/datadog-static-analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/datadog-static-analysis.yml b/.github/workflows/datadog-static-analysis.yml index 3a18cadfca808..1c32b7355cacf 100644 --- a/.github/workflows/datadog-static-analysis.yml +++ b/.github/workflows/datadog-static-analysis.yml @@ -11,7 +11,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Check code meets quality and security standards id: datadog-static-analysis - uses: DataDog/datadog-static-analyzer-github-action@v1 + uses: DataDog/datadog-static-analyzer-github-action@2707598b1182dce1d1792186477b5b4132338e1c # v1.2.3 with: dd_api_key: ${{ secrets.DD_API_KEY }} dd_app_key: ${{ secrets.DD_STATIC_ANALYSIS_APP_KEY }} From 52d6d4d49ff720d38bef5dbafe29e596624ce530 Mon Sep 17 00:00:00 2001 From: Juanpe Araque Date: Mon, 25 Aug 2025 11:50:45 +0100 Subject: [PATCH 2/4] Update test-target to set datadog service per integration (#21147) * Update test-target to set the service per test target * Update service name --- .github/workflows/test-target.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-target.yml b/.github/workflows/test-target.yml index 7d6ada4e0bca6..009307c0e74a8 100644 --- a/.github/workflows/test-target.yml +++ b/.github/workflows/test-target.yml @@ -106,7 +106,7 @@ jobs: TEST_RESULTS_BASE_DIR: "test-results" # Tracing to monitor our test suite DD_ENV: "ci" - DD_SERVICE: "ddev-integrations-${{ inputs.repo }}" + DD_SERVICE: "${{ inputs.target }}-integrations-${{ inputs.repo }}" DD_TAGS: "team:agent-integrations,platform:${{ inputs.platform }},integration:${{ inputs.target }}" DD_TRACE_ANALYTICS_ENABLED: "true" # Capture traces for a separate job to do the submission From 1a9c7b00c5407283f9f5e6cf707d978b7f679d2e Mon Sep 17 00:00:00 2001 From: Gustavo Mora Date: Mon, 25 Aug 2025 15:04:01 +0200 Subject: [PATCH 3/4] feat: making Jamf Pro docs public (#21133) --- jamf_pro/manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jamf_pro/manifest.json b/jamf_pro/manifest.json index 1bc90a80eba66..9ad016205f445 100644 --- a/jamf_pro/manifest.json +++ b/jamf_pro/manifest.json @@ -2,7 +2,7 @@ "manifest_version": "2.0.0", "app_uuid": "02ca0a87-f9e7-46e3-989d-9ac8b3654ac4", "app_id": "jamf-pro", - "display_on_public_website": false, + "display_on_public_website": true, "tile": { "overview": "README.md#Overview", "configuration": "README.md#Setup", From 7286fd7aacf3403f2467718fa888d913fa58fd3f Mon Sep 17 00:00:00 2001 From: Juanpe Araque Date: Mon, 25 Aug 2025 14:32:17 +0100 Subject: [PATCH 4/4] Create hatch command wrappers (#21135) * Create wrapper around hatch commands and refactor `ddev env test` to use it. This simplifies the logic for environment selection. * Fix changelog number --- ddev/changelog.d/21135.added | 1 + ddev/src/ddev/cli/env/test.py | 54 ++++++---- ddev/src/ddev/cli/test/__init__.py | 2 +- ddev/src/ddev/e2e/run.py | 2 +- ddev/src/ddev/testing/hatch.py | 12 --- ddev/src/ddev/utils/hatch.py | 122 +++++++++++++++++++++ ddev/tests/cli/env/test_test.py | 27 ++++- ddev/tests/testing/test_hatch.py | 15 --- ddev/tests/utils/test_hatch.py | 166 +++++++++++++++++++++++++++++ 9 files changed, 346 insertions(+), 55 deletions(-) create mode 100644 ddev/changelog.d/21135.added delete mode 100644 ddev/src/ddev/testing/hatch.py create mode 100644 ddev/src/ddev/utils/hatch.py delete mode 100644 ddev/tests/testing/test_hatch.py create mode 100644 ddev/tests/utils/test_hatch.py diff --git a/ddev/changelog.d/21135.added b/ddev/changelog.d/21135.added new file mode 100644 index 0000000000000..2681ce783bc53 --- /dev/null +++ b/ddev/changelog.d/21135.added @@ -0,0 +1 @@ +Add a utils.hatch module to centralize hatch operations \ No newline at end of file diff --git a/ddev/src/ddev/cli/env/test.py b/ddev/src/ddev/cli/env/test.py index 3a7e64f015631..148d674b04495 100644 --- a/ddev/src/ddev/cli/env/test.py +++ b/ddev/src/ddev/cli/env/test.py @@ -9,6 +9,7 @@ if TYPE_CHECKING: from ddev.cli.application import Application + from ddev.utils.hatch import Environment @click.command('test') @@ -72,6 +73,8 @@ def test_command( \b https://datadoghq.dev/integrations-core/testing/ """ + from functools import partial + from ddev.cli.env.start import start from ddev.cli.env.stop import stop from ddev.cli.test import test @@ -79,6 +82,7 @@ def test_command( from ddev.e2e.config import EnvDataStorage from ddev.e2e.constants import E2EMetadata from ddev.utils.ci import running_in_ci + from ddev.utils.hatch import HatchCommandError, list_environment_names from ddev.utils.structures import EnvVars app: Application = ctx.obj @@ -93,29 +97,19 @@ def test_command( if environment == 'active': env_names = active_envs else: - import json - import sys - - with integration.path.as_cwd(): - env_data_output = app.platform.check_command_output( - [sys.executable, '-m', 'hatch', '--no-color', '--no-interactive', 'env', 'show', '--json'] + try: + env_names = list_environment_names( + app.platform, + integration, + filters=[ + is_e2e_environment, + partial(uses_python_version, python_filter=python_filter), + partial(uses_platform, platform=app.platform.name), + partial(is_selected_environment, environment_name=environment), + ], ) - try: - environments = json.loads(env_data_output) - except json.JSONDecodeError: - app.abort(f'Failed to parse environments for `{integration.name}`:\n{repr(env_data_output)}') - - no_python_filter = python_filter is None - all_environments = environment == 'all' - - env_names = [ - name - for name, data in environments.items() - if data.get('e2e-env', False) - and (not data.get('platforms') or app.platform.name in data['platforms']) - and (no_python_filter or data.get('python') == python_filter) - and (name == environment or all_environments) - ] + except HatchCommandError as error: + app.abort(f'Failed to list environments for `{integration.name}`:\n{error}') if not env_names: app.display_info(f"Selected target {integration.name!r} disabled by e2e-env option.") @@ -155,3 +149,19 @@ def test_command( ) finally: ctx.invoke(stop, intg_name=intg_name, environment=env_name, ignore_state=env_active) + + +def is_e2e_environment(environment: Environment) -> bool: + return environment.e2e_env + + +def uses_python_version(environment: Environment, python_filter: str | None) -> bool: + return python_filter is None or environment.python == python_filter + + +def uses_platform(environment: Environment, platform: str) -> bool: + return not environment.platforms or platform in environment.platforms + + +def is_selected_environment(environment: Environment, environment_name: str) -> bool: + return environment.name == environment_name or environment_name == 'all' diff --git a/ddev/src/ddev/cli/test/__init__.py b/ddev/src/ddev/cli/test/__init__.py index d568ef36d91e3..3fdadc2b5afd4 100644 --- a/ddev/src/ddev/cli/test/__init__.py +++ b/ddev/src/ddev/cli/test/__init__.py @@ -130,8 +130,8 @@ def test( import sys from ddev.testing.constants import EndToEndEnvVars, TestEnvVars - from ddev.testing.hatch import get_hatch_env_vars from ddev.utils.ci import running_in_ci + from ddev.utils.hatch import get_hatch_env_vars if target_spec is None: target_spec = 'changed' diff --git a/ddev/src/ddev/e2e/run.py b/ddev/src/ddev/e2e/run.py index 7e0858a30254a..2f9053de4e42a 100644 --- a/ddev/src/ddev/e2e/run.py +++ b/ddev/src/ddev/e2e/run.py @@ -8,7 +8,7 @@ from typing import Generator from ddev.e2e.constants import E2EEnvVars -from ddev.testing.hatch import get_hatch_env_vars +from ddev.utils.hatch import get_hatch_env_vars from ddev.utils.structures import EnvVars diff --git a/ddev/src/ddev/testing/hatch.py b/ddev/src/ddev/testing/hatch.py deleted file mode 100644 index c5cebc53ae9a4..0000000000000 --- a/ddev/src/ddev/testing/hatch.py +++ /dev/null @@ -1,12 +0,0 @@ -# (C) Datadog, Inc. 2023-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) -def get_hatch_env_vars(*, verbosity: int) -> dict[str, str]: - env_vars = {} - - if verbosity > 0: - env_vars['HATCH_VERBOSE'] = str(verbosity) - elif verbosity < 0: - env_vars['HATCH_QUIET'] = str(abs(verbosity)) - - return env_vars diff --git a/ddev/src/ddev/utils/hatch.py b/ddev/src/ddev/utils/hatch.py new file mode 100644 index 0000000000000..6b8ada273eebd --- /dev/null +++ b/ddev/src/ddev/utils/hatch.py @@ -0,0 +1,122 @@ +# (C) Datadog, Inc. 2025-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from __future__ import annotations + +from typing import TYPE_CHECKING, Protocol, overload + +from pydantic import BaseModel, Field, RootModel, model_validator + +if TYPE_CHECKING: + from collections.abc import Collection + from typing import Any, Literal + + from ddev.integration.core import Integration + + from .platform import Platform + + +class HatchCommandError(Exception): + pass + + +class Environment(BaseModel): + """Represents a single environment configuration.""" + + name: str + type: str + dependencies: list[str] = Field(default_factory=list) + test_env: bool = Field(alias='test-env') + e2e_env: bool = Field(alias='e2e-env') + benchmark_env: bool = Field(alias='benchmark-env') + latest_env: bool = Field(alias='latest-env') + python: str | None = None + scripts: dict[str, list[str]] = Field(default_factory=dict) + platforms: list[str] = Field(default_factory=list) + pre_install_commands: list[str] = Field(default_factory=list, alias='pre-install-commands') + post_install_commands: list[str] = Field(default_factory=list, alias='post-install-commands') + skip_install: bool = Field(False, alias='skip-install') + + +class HatchEnvironmentConfiguration(RootModel[list[Environment]]): + """ + A root model that parses the top-level dictionary returned by the `hatch env show --json` command + into a list of Environment models. + """ + + @model_validator(mode='before') + @classmethod + def transform_dict_to_list_with_name(cls, data: object) -> object: + if isinstance(data, list): + return data + + if isinstance(data, dict): + return [Environment(**value, name=key) for key, value in data.items()] + + raise ValueError(f'Invalid data type: {type(data)}. Expected a list or a dictionary.') + + +class EnvironmentFilter(Protocol): + def __call__(self, environment: Environment) -> bool: ... + + +@overload +def env_show(platform: Platform, integration: Integration, as_json: Literal[True]) -> dict[str, Any]: ... + + +@overload +def env_show(platform: Platform, integration: Integration) -> dict[str, Any]: ... + + +@overload +def env_show(platform: Platform, integration: Integration, as_json: Literal[False]) -> str: ... + + +def env_show(platform: Platform, integration: Integration, as_json: bool = True) -> dict[str, Any] | str: + import json + import sys + + with integration.path.as_cwd(): + command = [sys.executable, '-m', 'hatch', '--no-color', '--no-interactive', 'env', 'show'] + if as_json: + command.append('--json') + + env_data_output = platform.check_command_output(command) + + try: + if as_json: + return json.loads(env_data_output) + return env_data_output + except json.JSONDecodeError as error: + raise HatchCommandError( + f'Failed to parse environments for {integration.name!r}: {env_data_output!r}' + ) from error + + +def list_environment_names( + platform: Platform, integration: Integration, filters: Collection[EnvironmentFilter], match_all: bool = True +) -> list[str]: + """ + List the names of the environments that match the given filters. + + If `match_all` is True, all filters must match. If False, any filter can match. + """ + hatch_output = env_show(platform, integration) + matching_rule = all if match_all else any + + return [ + env.name + for env in HatchEnvironmentConfiguration.model_validate(hatch_output).root + if matching_rule(filter(env) for filter in filters) + ] + + +def get_hatch_env_vars(*, verbosity: int) -> dict[str, str]: + env_vars = {} + + if verbosity > 0: + env_vars['HATCH_VERBOSE'] = str(verbosity) + elif verbosity < 0: + env_vars['HATCH_QUIET'] = str(abs(verbosity)) + + return env_vars diff --git a/ddev/tests/cli/env/test_test.py b/ddev/tests/cli/env/test_test.py index 8876124af8862..5191f48b5f47e 100644 --- a/ddev/tests/cli/env/test_test.py +++ b/ddev/tests/cli/env/test_test.py @@ -16,6 +16,21 @@ from tests.helpers.mocks import MockPopen from tests.helpers.runner import CliRunner +BASE_ENV_CONFIG = { + 'type': 'virtual', + 'dependencies': [], + 'test-env': True, + 'e2e-env': True, + 'benchmark-env': True, + 'latest-env': True, + 'python': '3.12', + 'scripts': {}, + 'platforms': [], + 'pre-install-commands': [], + 'post-install-commands': [], + 'skip-install': False, +} + def setup( mocker: MockerFixture, @@ -96,7 +111,7 @@ def test_env_vars_repo( predicate: Callable[[Result], bool], mock_commands: tuple[MockType, MockType, MockType], ): - setup(mocker, write_result_file, hatch_json_output={'py3.12': {'e2e-env': e2e_env}}) + setup(mocker, write_result_file, hatch_json_output={'py3.12': {**BASE_ENV_CONFIG, 'e2e-env': e2e_env}}) mocker.patch.object(EnvData, 'read_metadata', return_value={}) result = ddev('env', 'test', 'postgres', 'py3.12') @@ -119,7 +134,11 @@ def test_environment_runs_for_enabled_environments( setup( mocker, write_result_file, - hatch_json_output={'py3.12': {'e2e-env': True}, 'py3.13': {'e2e-env': False}, 'py3.13-v1': {'e2e-env': True}}, + hatch_json_output={ + 'py3.12': BASE_ENV_CONFIG, + 'py3.13': {**BASE_ENV_CONFIG, 'e2e-env': False}, + 'py3.13-v1': BASE_ENV_CONFIG, + }, ) with mocker.patch.object(EnvData, 'read_metadata', return_value={}): result = ddev('env', 'test', 'postgres', environment) @@ -145,7 +164,7 @@ def test_runningin_ci_triggers_all_environments_when_not_supplied( mocker: MockerFixture, mock_commands: tuple[MockType, MockType, MockType], ): - setup(mocker, write_result_file, hatch_json_output={'py3.12': {'e2e-env': True}, 'py3.13': {'e2e-env': True}}) + setup(mocker, write_result_file, hatch_json_output={'py3.12': BASE_ENV_CONFIG, 'py3.13': BASE_ENV_CONFIG}) mocker.patch('ddev.utils.ci.running_in_ci', return_value=True) with mocker.patch.object(EnvData, 'read_metadata', return_value={}): @@ -161,7 +180,7 @@ def test_run_only_active_environments_when_not_running_in_ci_and_active_environm mocker: MockerFixture, mock_commands: tuple[MockType, MockType, MockType], ): - setup(mocker, write_result_file, hatch_json_output={'py3.12': {'e2e-env': True}, 'py3.13': {'e2e-env': True}}) + setup(mocker, write_result_file, hatch_json_output={'py3.12': BASE_ENV_CONFIG, 'py3.13': BASE_ENV_CONFIG}) mocker.patch('ddev.utils.ci.running_in_ci', return_value=False) with ( diff --git a/ddev/tests/testing/test_hatch.py b/ddev/tests/testing/test_hatch.py deleted file mode 100644 index 7cbe05a5830b6..0000000000000 --- a/ddev/tests/testing/test_hatch.py +++ /dev/null @@ -1,15 +0,0 @@ -# (C) Datadog, Inc. 2023-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) -from ddev.testing.hatch import get_hatch_env_vars - - -class TestGetHatchEnvVars: - def test_no_verbosity(self): - assert not get_hatch_env_vars(verbosity=0) - - def test_increased_verbosity(self): - assert get_hatch_env_vars(verbosity=1) == {'HATCH_VERBOSE': '1'} - - def test_decreased_verbosity(self): - assert get_hatch_env_vars(verbosity=-1) == {'HATCH_QUIET': '1'} diff --git a/ddev/tests/utils/test_hatch.py b/ddev/tests/utils/test_hatch.py new file mode 100644 index 0000000000000..700fd7e49f3ee --- /dev/null +++ b/ddev/tests/utils/test_hatch.py @@ -0,0 +1,166 @@ +# (C) Datadog, Inc. 2025-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import json + +import pytest +from pydantic import ValidationError +from pytest_mock import MockerFixture + +from ddev.integration.core import Integration +from ddev.utils import hatch +from ddev.utils.platform import Platform + + +@pytest.fixture +def mock_integration(mocker: MockerFixture) -> Integration: + mock_integration = mocker.MagicMock(spec=Integration) + mock_integration.path.as_cwd.return_value = mocker.MagicMock( + __enter__=mocker.MagicMock(), __exit__=mocker.MagicMock(return_value=None) + ) + mock_integration.name = "my-integration" + return mock_integration + + +def platform(mocker: MockerFixture, output: str) -> Platform: + platform = mocker.MagicMock(spec=Platform) + platform.check_command_output.return_value = output + return platform + + +def test_no_verbosity(): + assert not hatch.get_hatch_env_vars(verbosity=0) + + +def test_increased_verbosity(): + assert hatch.get_hatch_env_vars(verbosity=1) == {'HATCH_VERBOSE': '1'} + + +def test_decreased_verbosity(): + assert hatch.get_hatch_env_vars(verbosity=-1) == {'HATCH_QUIET': '1'} + + +def test_hatch_environment_configuration_from_dict(): + data = { + "default": { + "type": "virtual", + "dependencies": ["dep1"], + "test-env": True, + "e2e-env": False, + "benchmark-env": False, + "latest-env": False, + } + } + config = hatch.HatchEnvironmentConfiguration.model_validate(data) + assert len(config.root) == 1 + env = config.root[0] + assert env.name == "default" + assert env.type == "virtual" + assert env.dependencies == ["dep1"] + assert env.test_env is True + + +def test_hatch_environment_configuration_from_list(): + data = [ + { + "name": "default", + "type": "virtual", + "dependencies": ["dep1"], + "test-env": True, + "e2e-env": False, + "benchmark-env": False, + "latest-env": False, + } + ] + config = hatch.HatchEnvironmentConfiguration.model_validate(data) + assert len(config.root) == 1 + env = config.root[0] + assert env.name == "default" + + +def test_hatch_environment_configuration_invalid_data(): + with pytest.raises(ValidationError): + hatch.HatchEnvironmentConfiguration.model_validate("invalid") + + +def test_env_show_json(mocker: MockerFixture, mock_integration: Integration): + mock_platform = platform(mocker, '{"key": "value"}') + result = hatch.env_show(mock_platform, mock_integration, as_json=True) + + assert result == {"key": "value"} + mock_platform.check_command_output.assert_called_once() + cmd = mock_platform.check_command_output.call_args[0][0] + assert '--json' in cmd + + +def test_env_show_string(mocker: MockerFixture, mock_integration: Integration): + mock_platform = platform(mocker, 'some output') + result = hatch.env_show(mock_platform, mock_integration, as_json=False) + + assert result == "some output" + mock_platform.check_command_output.assert_called_once() + cmd = mock_platform.check_command_output.call_args[0][0] + assert '--json' not in cmd + + +def test_env_show_invalid_json(mocker: MockerFixture, mock_integration: Integration): + mock_platform = platform(mocker, 'invalid json') + with pytest.raises(hatch.HatchCommandError): + hatch.env_show(mock_platform, mock_integration, as_json=True) + + +HATCH_OUTPUT = { + "linux_e2e_no_python": { + "type": "virtual", + "dependencies": [], + "test-env": True, + "e2e-env": False, + "benchmark-env": False, + "latest-env": True, + "platforms": ["linux"], + }, + "no_platform_no_e2e_py3.8": { + "type": "virtual", + "dependencies": [], + "test-env": True, + "e2e-env": False, + "benchmark-env": False, + "latest-env": False, + "python": "3.8", + }, + "no_platform_e2e_no_python": { + "type": "virtual", + "dependencies": [], + "test-env": False, + "e2e-env": True, + "benchmark-env": False, + "latest-env": False, + }, +} + + +def test_list_environment_names_match_all(mocker: MockerFixture, mock_integration: Integration): + mock_platform = platform(mocker, json.dumps(HATCH_OUTPUT)) + filters = [lambda environment: environment.test_env, lambda environment: "linux" in environment.platforms] + + names = hatch.list_environment_names(mock_platform, mock_integration, filters, match_all=True) + + assert names == ["linux_e2e_no_python"] + + +def test_list_environment_names_match_any(mocker: MockerFixture, mock_integration: Integration): + mock_platform = platform(mocker, json.dumps(HATCH_OUTPUT)) + filters = [lambda environment: environment.e2e_env, lambda environment: environment.python == "3.8"] + + names = hatch.list_environment_names(mock_platform, mock_integration, filters, match_all=False) + + assert sorted(names) == sorted(["no_platform_no_e2e_py3.8", "no_platform_e2e_no_python"]) + + +def test_list_environment_names_no_match(mocker: MockerFixture, mock_integration: Integration): + mock_platform = platform(mocker, json.dumps(HATCH_OUTPUT)) + filters = [lambda environment: environment.benchmark_env] + + names = hatch.list_environment_names(mock_platform, mock_integration, filters) + + assert names == []