Improve our CI visibility reporting and streamline test-target (DataDog#22355)

* Use official datadog GH action to monitor CI visibility

* Add changelog

* Add some debugging for failing tests

* Remove action and ensure ddtrace is a depdnency of the default environment

* Revert changes for debugging in test_agent_packages

* Remove profiling

* Remove trace analytics

* Disable subprocess tracing

* Try removing git metadta gathering

* Debug messages to understand git issues

* Fix formattingh

* Fix error in debug code

* Fix formatting

* Make sure we unshallwo the repo

* Also unshallwo for windows setup for ddev

* Add test environment name to differentiate between e2e and unit tests

* Update documentation

* Add ci visibility to e2e tests as well

* Try reducing the size of test-target

* Streamline test-target to reduce the object footprint on actions

* Ensure python version set to 3.13 when input is an empty string

* Ensure python version is updatable through the py upgrade script

* Print actual ddev command run when executing tests

* Update ddev env test command to respect trace enabled option

* Fix update_py_config script patterns

* Fix AppEnvVar for enable tracing

* Modify integration tag to target

* Tag jobs with the same tags as we tag tests

* Add target_env tag to the test workflow

* Make bash script more robust

* Avoid running flaky tests in pr-all

* Fix handling of pytest args

* Address comments

Author: AAraKKe

.ddev/ci/scripts/ddev/linux/55_fetch_master.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -ex
+
+# Only required on non-master branches
+# To benefit from ci visibility, ddtrace needs to explore git and the local clone we do in ddev tests
+# from a shallow repo can break it. Lets get the full repo so we have a proper repo to clone from locally.
+
+if [ "$(git rev-parse --is-shallow-repository)" = "true" ]; then
+  echo "Repository is shallow. Unshallowing to get full history..."
+  # --unshallow: Downloads the rest of the commit history
+  # --tags: Ensures we get tags (useful for versioning)
+  # -f: Force update references if needed
+  git fetch --unshallow --tags -f origin
+fi
+
+# 2. Your original logic: Ensure we have the master branch reference locally
+# (Unshallowing gets the history, but doesn't necessarily create the local 'master' ref)
+if [[ "$GITHUB_REF_NAME" != "master" ]]; then
+  git fetch origin master:master
+fi
+
+set +ex

.ddev/ci/scripts/ddev/linux/55_fetch_origin.sh

@@ -0,0 +1,15 @@
+@echo on
+
+:: 1. Check if the repo is shallow by capturing the output of the git command
+FOR /F "tokens=*" %%g IN ('git rev-parse --is-shallow-repository') do (SET IS_SHALLOW=%%g)
+
+:: 2. If it is shallow ("true"), fetch the full history
+IF "%IS_SHALLOW%"=="true" (
+  ECHO Repository is shallow. Unshallowing to get full history...
+  git fetch --unshallow --tags -f origin
+)
+
+:: 3. Only required on non-master branches (Your original logic)
+if "%GITHUB_REF_NAME%" NEQ "master" (
+  git fetch origin master:master
+)

.ddev/ci/scripts/ddev/windows/55_fetch_master.bat

@@ -0,0 +1,41 @@
+name: 'Tag Job with Datadog CI'
+description: 'Tags the current job with custom tags for CI Visibility in Datadog'
+
+inputs:
+  tags:
+    description: 'Comma-separated list of tags to add (e.g., "custom_tag:value,another_tag:value2")'
+    required: true
+  dd_api_key:
+    description: 'Datadog API key'
+    required: true
+  dd_site:
+    description: 'Datadog site (e.g., datadoghq.com, datad0g.com)'
+    required: false
+    default: 'datadoghq.com'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Set up Node.js
+      uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
+      with:
+        node-version: '20'
+
+    - name: Install datadog-ci and tag job
+      shell: bash
+      env:
+        DATADOG_API_KEY: ${{ inputs.dd_api_key }}
+        DATADOG_SITE: ${{ inputs.dd_site }}
+        TAGS: ${{ inputs.tags }}
+      run: |
+        npm install -g @datadog/datadog-ci
+
+        # Convert comma-separated tags to --tags flag format
+        # e.g., "tag1:value1,tag2:value2" becomes "--tags tag1:value1 --tags tag2:value2"
+        TAGS_ARGS=""
+        IFS=',' read -ra TAG_ARRAY <<< "$TAGS"
+        for tag in "${TAG_ARRAY[@]}"; do
+          TAGS_ARGS="$TAGS_ARGS --tags $tag"
+        done
+
+        datadog-ci tag --level job $TAGS_ARGS

.ddev/ci/scripts/ddev/windows/55_fetch_origin.bat

@@ -49,11 +49,3 @@ jobs:
 
     uses: ./.github/workflows/test-results-master.yml
     secrets: inherit
-
-  submit-traces:
-    needs:
-    - test
-    if: success() || failure()
-
-    uses: ./.github/workflows/submit-traces.yml
-    secrets: inherit

.ddev/ci/scripts/traces.py

@@ -47,11 +47,3 @@ jobs:
 
     uses: ./.github/workflows/test-results-master.yml
     secrets: inherit
-
-  submit-traces:
-    needs:
-    - test
-    if: success() || failure()
-
-    uses: ./.github/workflows/submit-traces.yml
-    secrets: inherit

.github/actions/tag-job/action.yml

@@ -96,11 +96,3 @@ jobs:
         use_oidc: true
         directory: coverage-reports
         fail_ci_if_error: false
-
-  submit-traces:
-    needs:
-    - test
-    if: success() || failure()
-
-    uses: ./.github/workflows/submit-traces.yml
-    secrets: inherit

.github/workflows/flaky-tests-windows.yml

@@ -93,11 +93,3 @@ jobs:
         use_oidc: true
         directory: coverage-reports
         fail_ci_if_error: false
-
-  submit-traces:
-    needs:
-    - test
-    if: success() || failure()
-
-    uses: ./.github/workflows/submit-traces.yml
-    secrets: inherit