[pull] master from DataDog:master

.ddev/config.toml

@@ -12,6 +12,12 @@ openstack_controller = "OpenStack Controller"
 pulsar = "Pulsar"
 teamcity = "TeamCity"
 win32_event_log = "Windows Event Log"
+krakend = "KrakenD"
+lustre = "Lustre"
+
+[overrides.metrics-prefix]
+krakend = "krakend.api."
+lustre = "lustre."
 
 [overrides.ci.ddev]
 platforms = ["linux", "windows"]

.gitlab-ci.yml

@@ -28,8 +28,8 @@ validate-log-integrations:
   stage: validate
   needs: []
   image: $VALIDATE_LOG_INTGS
-  only:
-  - schedules
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
   variables:
     INTEGRATIONS_CORE_ROOT: $CI_PROJECT_DIR
   script:
@@ -52,8 +52,8 @@ notify-slack:
     - validate-log-integrations
   stage: notify
   image: $NOTIFIER_IMAGE
-  only:
-  - schedules
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
   cache:
     <<: *slack-cache
   script:
@@ -73,9 +73,9 @@ notify-slack:
 notify-failed-pipeline:
   stage: notify
   image: $NOTIFIER_IMAGE
-  only:
-  - master
-  when: on_failure
+  rules:
+    - if: $CI_COMMIT_BRANCH == "master"
+      when: on_failure
   cache:
     <<: *slack-cache
   script:
@@ -89,11 +89,11 @@ notify-failed-pipeline:
 release-auto:
   stage: release
   image: $TAGGER_IMAGE
-  only:
-    - master
-    - /^\d+\.\d+\.x$/
-  except:
-    - schedules
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: never
+    - if: $CI_COMMIT_BRANCH == "master"
+    - if: $CI_COMMIT_BRANCH =~ /^\d+\.\d+\.x$/
   script:
     - ddev --version
     - ddev config override
@@ -105,11 +105,11 @@ release-auto:
 release-manual:
   stage: release
   image: $TAGGER_IMAGE
-  only:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: never
     # Integration release tags e.g. any_check-X.Y.Z-rc.N
-    - /.*-\d+\.\d+\.\d+(-(rc|pre|alpha|beta)\.\d+)?$/
-  except:
-  - schedules
+    - if: $CI_COMMIT_TAG =~ /.*-\d+\.\d+\.\d+(-(rc|pre|alpha|beta)\.\d+)?$/
   script:
     # Get tagger info
     - tagger=$(git for-each-ref refs/tags/$CI_COMMIT_TAG  --format='%(taggername) %(taggeremail)')

datadog_checks_dev/changelog.d/21784.fixed

@@ -0,0 +1 @@
+Add a fallback mechanism when mount_logs is True and no manifest is present in the integration

datadog_checks_dev/datadog_checks/dev/spec.py

@@ -8,23 +8,45 @@
 from .utils import file_exists, path_join, read_file
 
 
-def load_spec(check_root):
+def load_spec(check_root: str):
     spec_path = get_spec_path(check_root)
     return yaml.safe_load(read_file(spec_path))
 
 
-def get_spec_path(check_root):
-    manifest = json.loads(read_file(path_join(check_root, 'manifest.json')))
+def get_spec_path(check_root: str) -> str:
+    sources = [
+        __get_spec_path_from_manifest,
+        __get_spec_path_from_common_location,
+    ]
+
+    for source in sources:
+        if (spec_path := source(check_root)) is not None:
+            if not file_exists(spec_path):
+                raise ValueError('No config spec found')
+
+            return spec_path
+
+    raise ValueError('No config spec found')
+
+
+def __get_spec_path_from_common_location(check_root: str) -> str | None:
+    spec_path = path_join(check_root, 'assets', 'configuration', 'spec.yaml')
+    return spec_path if file_exists(spec_path) else None
+
+
+def __get_spec_path_from_manifest(check_root: str) -> str | None:
+    manifest_path = path_join(check_root, 'manifest.json')
+    if not file_exists(manifest_path):
+        return None
+
+    manifest = json.loads(read_file(manifest_path))
     assets = manifest.get('assets', {})
     if 'integration' in assets:
         relative_spec_path = assets['integration'].get('configuration', {}).get('spec', '')
     else:
         relative_spec_path = assets.get('configuration', {}).get('spec', '')
-    if not relative_spec_path:
-        raise ValueError('No config spec defined')
 
-    spec_path = path_join(check_root, *relative_spec_path.split('/'))
-    if not file_exists(spec_path):
-        raise ValueError('No config spec found')
+    if not relative_spec_path:
+        return None
 
-    return spec_path
+    return path_join(check_root, *relative_spec_path.split('/'))

ddev/changelog.d/21818.fixed

@@ -0,0 +1 @@
+Support CI validation for workflows using pinned commit SHAs instead of @master

ddev/changelog.d/21820.fixed

@@ -0,0 +1 @@
+Allow the `validate metadata` command to get the metrics-prefix from the repo overrides config in case the manifest file does not exist

ddev/src/ddev/cli/validate/ci.py

@@ -56,10 +56,45 @@ def ci(app: Application, sync: bool):
 
     is_core = app.repo.name == 'core'
     is_marketplace = app.repo.name == 'marketplace'
+
+    # For non-core repos, extract the workflow reference from existing workflow files
+    # This allows using either @master or @<commit-sha>
+    workflow_ref = 'master'  # default
+    windows_workflow_ref = 'master'  # default
+    if not is_core:
+        jobs_workflow_path_temp = app.repo.path / '.github' / 'workflows' / 'test-all.yml'
+        windows_jobs_workflow_path_temp = app.repo.path / '.github' / 'workflows' / 'test-all-windows.yml'
+
+        # Extract reference from Linux workflow file
+        if jobs_workflow_path_temp.is_file():
+            existing_workflow = jobs_workflow_path_temp.read_text()
+            # Look for pattern like: DataDog/integrations-core/.github/workflows/test-target.yml@<ref>
+            # <ref> can be a branch name (alphanumeric + dots/dashes/underscores) or commit SHA (hex only)
+            match = re.search(
+                r'DataDog/integrations-core/\.github/workflows/test-target\.yml@([a-zA-Z0-9_.-]+)', existing_workflow
+            )
+            if match:
+                workflow_ref = match.group(1)
+
+        # Extract reference from Windows workflow file
+        if windows_jobs_workflow_path_temp.is_file():
+            existing_windows_workflow = windows_jobs_workflow_path_temp.read_text()
+            match = re.search(
+                r'DataDog/integrations-core/\.github/workflows/test-target\.yml@([a-zA-Z0-9_.-]+)',
+                existing_windows_workflow,
+            )
+            if match:
+                windows_workflow_ref = match.group(1)
+
     test_workflow = (
         './.github/workflows/test-target.yml'
         if app.repo.name == 'core'
-        else 'DataDog/integrations-core/.github/workflows/test-target.yml@master'
+        else f'DataDog/integrations-core/.github/workflows/test-target.yml@{workflow_ref}'
+    )
+    windows_test_workflow = (
+        './.github/workflows/test-target.yml'
+        if app.repo.name == 'core'
+        else f'DataDog/integrations-core/.github/workflows/test-target.yml@{windows_workflow_ref}'
     )
     jobs_workflow_path = app.repo.path / '.github' / 'workflows' / 'test-all.yml'
     windows_jobs_workflow_path = app.repo.path / '.github' / 'workflows' / 'test-all-windows.yml'
@@ -145,7 +180,9 @@ def ci(app: Application, sync: bool):
         job_id = hashlib.sha256(config['job-name'].encode('utf-8')).hexdigest()[:7]
         job_id = f'j{job_id}'
 
-        job_config = {'uses': test_workflow, 'with': config, 'secrets': 'inherit'}
+        # Use the appropriate workflow reference based on platform
+        workflow_to_use = windows_test_workflow if 'windows' in data['platform'] else test_workflow
+        job_config = {'uses': workflow_to_use, 'with': config, 'secrets': 'inherit'}
         if 'target-env' in config:
             job_config['strategy'] = {
                 'matrix': {'target-env': data['target-env']},

ddev/src/ddev/cli/validate/metadata.py

@@ -90,7 +90,14 @@ def metadata(app: Application, integrations: tuple[str, ...], check_duplicates:
         if current_check.name.startswith('datadog_checks_'):
             continue
 
-        metric_prefix = current_check.manifest.get("/assets/integration/metrics/prefix", "")
+        metric_prefix = current_check.manifest.get("/assets/integration/metrics/prefix", "") or app.repo.config.get(
+            f"/overrides/metrics-prefix/{current_check.name}", ""
+        )
+
+        if not metric_prefix and current_check.name not in metadata_utils.PROVIDER_INTEGRATIONS:
+            errors = True
+            error_message += f'{current_check.name}: metric_prefix does not exist in manifest or overrides.\n'
+
         metadata_file = current_check.metrics_file
 
         # To make logging less verbose, common errors are counted for current check
@@ -101,7 +108,6 @@ def metadata(app: Application, integrations: tuple[str, ...], check_duplicates:
         duplicate_short_name_set: set = set()
         duplicate_description_set: set = set()
 
-        metric_prefix_error_shown = False
         if metadata_file.stat().st_size == 0:
             errors = True
 
@@ -173,12 +179,6 @@ def metadata(app: Application, integrations: tuple[str, ...], check_duplicates:
                     'metric_name'
                 ].startswith(metric_prefix):
                     metric_prefix_count[prefix] += 1
-            else:
-                errors = True
-                if not metric_prefix_error_shown and current_check.name not in metadata_utils.PROVIDER_INTEGRATIONS:
-                    metric_prefix_error_shown = True
-
-                    error_message += f'{current_check.name}:{line} metric_prefix does not exist in manifest.\n'
 
             # metric_type header
             if row['metric_type'] and row['metric_type'] not in metadata_utils.VALID_METRIC_TYPE:
@@ -284,7 +284,7 @@ def metadata(app: Application, integrations: tuple[str, ...], check_duplicates:
             error_message += (
                 f"{current_check.name}: `{prefix}` appears {count} time(s) and does not match metric_prefix "
             )
-            error_message += "defined in the manifest.\n"
+            error_message += f"defined for this integration: {metric_prefix=}.\n"
 
         unsorted = set(app.repo.config.get('/overrides/validate/metrics/unsorted', []))
 

ddev/src/ddev/integration/core.py

@@ -94,7 +94,7 @@ def display_name(self) -> str:
 
     @cached_property
     def normalized_display_name(self) -> str:
-        display_name = self.manifest.get('/assets/integration/source_type_name', self.name)
+        display_name = self.display_name
         normalized_integration = re.sub("[^0-9A-Za-z-]", "_", display_name)
         normalized_integration = re.sub("_+", "_", normalized_integration)
         normalized_integration = normalized_integration.strip("_")

ddev/tests/cli/validate/test_metrics.py

@@ -175,7 +175,7 @@ def test_manifest_metric_prefix_dne(ddev, repository, helpers):
         └── Apache
             └── {outfile}
 
-                apache:2 metric_prefix does not exist in manifest.
+                apache: metric_prefix does not exist in manifest or overrides.
 
         Errors: 1
         """
@@ -577,7 +577,7 @@ def test_prefix_match(ddev, repository, helpers):
             └── {outfile}
 
                 apache: `invalid_metric_prefix` appears 1 time(s) and does not match
-                metric_prefix defined in the manifest.
+                metric_prefix defined for this integration: metric_prefix='apache.'.
 
         Errors: 1
         """