MDBF-1158: Implement MTR stdout log parsing

Related: MariaDB/mtr-log-collector#1

This patch can be merged without waiting for MariaDB/mtr-log-collector#1
. Clients will continue to use .xml files, ensuring backward compatibility.

A separate patch will update the clients to use `stdout.log` files.

Changes:
- Update mtr_reporter.sh to handle log files
- Introduce a dry-run option for local testing
- Make the service name more generic: `MTR_LOG_COLLECTOR_BASE_URL`
- Update `MTRReporter` to handle both XML and log formats; XML remains the default to avoid breaking existing clients

Author: RazvanLiviuVarzaru

configuration/builders/sequences/debug.py

@@ -1,7 +1,7 @@
 from pathlib import PurePath
 
 from configuration.builders.infra.runtime import BuildSequence, InContainer
-from configuration.builders.sequences.helpers import mtr_junit_reporter, save_mtr_logs
+from configuration.builders.sequences.helpers import mtr_reporter, save_mtr_logs
 from configuration.steps.base import StepOptions
 from configuration.steps.commands.compile import MAKE, CompileMakeCommand
 from configuration.steps.commands.configure import ConfigureMariaDBCMake
@@ -170,7 +170,7 @@ def openssl_fips(
     )
 
     sequence.add_step(
-        mtr_junit_reporter(
+        mtr_reporter(
             step_wrapping_fn=lambda step: InContainer(
                 docker_environment=config, step=step
             ),

configuration/builders/sequences/helpers.py

@@ -340,7 +340,7 @@ def add_test_suites_steps(
     )
 
     steps.append(
-        mtr_junit_reporter(
+        mtr_reporter(
             step_wrapping_fn=lambda step: InContainer(
                 docker_environment=config, step=step
             ),
@@ -373,7 +373,7 @@ def save_mtr_logs(
     )
 
 
-def mtr_junit_reporter(
+def mtr_reporter(
     step_wrapping_fn=lambda step: step,
 ):
     return step_wrapping_fn(

configuration/builders/sequences/mtr_extra_coverage.py

@@ -1,7 +1,7 @@
 from pathlib import PurePath
 
 from configuration.builders.infra.runtime import BuildSequence, InContainer
-from configuration.builders.sequences.helpers import mtr_junit_reporter, save_mtr_logs
+from configuration.builders.sequences.helpers import mtr_reporter, save_mtr_logs
 from configuration.steps.base import StepOptions
 from configuration.steps.commands.compile import MAKE, CompileMakeCommand
 from configuration.steps.commands.configure import ConfigureMariaDBCMake
@@ -120,7 +120,7 @@ def big_test(config, jobs):
     )
 
     sequence.add_step(
-        mtr_junit_reporter(
+        mtr_reporter(
             step_wrapping_fn=lambda step: InContainer(
                 docker_environment=config, step=step
             ),

configuration/builders/sequences/sanitizers.py

@@ -9,7 +9,7 @@
     get_mtr_normal_steps,
     get_mtr_s3_steps,
     get_mtr_spider_steps,
-    mtr_junit_reporter,
+    mtr_reporter,
     save_mtr_logs,
 )
 from configuration.steps.base import StepOptions
@@ -170,7 +170,7 @@ def asan_ubsan(
                     docker_environment=config, step=step
                 ),
             ),
-            mtr_junit_reporter(
+            mtr_reporter(
                 step_wrapping_fn=lambda step: InContainer(
                     docker_environment=config, step=step
                 ),
@@ -296,7 +296,7 @@ def msan(
                 docker_environment=config, step=step
             ),
         ),
-        mtr_junit_reporter(
+        mtr_reporter(
             step_wrapping_fn=lambda step: InContainer(
                 docker_environment=config, step=step
             ),

configuration/steps/commands/mtr.py

@@ -1,4 +1,5 @@
 import os
+from enum import Enum
 from pathlib import PurePath
 
 from buildbot.plugins import util
@@ -112,23 +113,51 @@ def _save_logs(self) -> str:
             """
 
 
+class ContentKind(str, Enum):
+    """Enumeration for the kind of content to report to the MTR log collector."""
+
+    XML = "xml"
+    LOG = "log"
+
+
 class MTRReporter(BashScriptCommand):
     """
-    A command to transfer all the MTR JUnit test results to the mtr_junit_collector service.
+    A command to report MTR test results to the MTR log collector.
     Attributes:
-        directory (PurePath): The directory containing the MTR test results.
+        workdir (PurePath): The working directory for the command.
+        content_kind (ContentKind): The kind of content to report (XML or LOG).
+        dry_run (bool): Whether to perform a dry run without actual upload.
+
+    XML files are searched in the workdir, without traversing subdirectories.
+
+    LOG files are searched in the path workdir/*/log/stdout.log
     """
 
-    JUNIT_COLLECTOR_BASE_URL = os.environ.get("JUNIT_COLLECTOR_BASE_URL")
+    MTR_LOG_COLLECTOR_BASE_URL = os.environ.get("MTR_LOG_COLLECTOR_BASE_URL")
 
-    def __init__(self, workdir: PurePath = PurePath(".")):
-        base_url = self.JUNIT_COLLECTOR_BASE_URL
+    def __init__(
+        self,
+        workdir: PurePath = PurePath("."),
+        content_kind: ContentKind = ContentKind.XML,
+        dry_run: bool = False,
+    ):
+        base_url = self.MTR_LOG_COLLECTOR_BASE_URL
         branch = util.Interpolate("%(prop:branch)s")
         revision = util.Interpolate("%(prop:revision)s")
         platform = util.Interpolate("%(prop:buildername)s")
         bbnum = util.Interpolate("%(prop:buildnumber)s")
         dir = "."
 
-        args = [base_url, branch, revision, platform, bbnum, dir]
+        args = [
+            base_url,
+            branch,
+            revision,
+            platform,
+            bbnum,
+            dir,
+            content_kind.value,
+            "1" if dry_run else "0",
+        ]
+
         super().__init__(script_name="mtr_reporter.sh", args=args, workdir=workdir)
         self.name = "Save test results for CrossReference"

configuration/steps/commands/scripts/mtr_reporter.sh

@@ -3,12 +3,14 @@
 set -euo pipefail
 
 # Input variables
-BASE_URL="$1"     # e.g., 100.64.101.1:9990
+BASE_URL="$1"        # e.g., 100.64.101.1:9990
 BRANCH="$2"
 REVISION="$3"
 PLATFORM="$4"
 BBNUM="$5"
-DIR="$6"          # Directory containing .xml files
+DIR="$6"             # Base directory containing either .xml files OR component/log/stdout.log
+CONTENT_KIND="$7"    # "xml" or "log"
+DRY_RUN="${8:-0}"    # "1" = print curl commands only, "0" = actually run (default)
 
 err() {
   set +x
@@ -25,59 +27,117 @@ bb_log_info() {
 UPLOAD_URL="${BASE_URL}/upload-test-results/"
 HEALTH_URL="${BASE_URL}/health"
 
-# Step 1: Health check before uploads
+# Validate mode inputs early
+case "$CONTENT_KIND" in
+  xml|log) ;;
+  *) err "Error: CONTENT_KIND must be 'xml' or 'log' (got: '$CONTENT_KIND')" ;;
+esac
+
+case "$DRY_RUN" in
+  0|1) ;;
+  *) err "Error: DRY_RUN must be 0 or 1 (got: '$DRY_RUN')" ;;
+esac
+
+# Step 1: Health check before uploads (skip if dry run)
 command -v curl >/dev/null || err "curl not found"
-bb_log_info "Checking service health at $HEALTH_URL..."
-if ! curl "$HEALTH_URL" \
-  --max-time 5 \
-  --retry 3 \
-  --retry-max-time 0 \
-  --retry-delay 5 \
-  --retry-connrefused \
-  --fail-with-body; then
-    err "Service health check failed. Aborting uploads."
+
+if [[ "$DRY_RUN" == "1" ]]; then
+  bb_log_info "DRY RUN enabled: skipping health check and uploads will not be executed."
+else
+  bb_log_info "Checking service health at $HEALTH_URL..."
+  if ! curl "$HEALTH_URL" \
+    --max-time 5 \
+    --retry 3 \
+    --retry-max-time 0 \
+    --retry-delay 5 \
+    --retry-connrefused \
+    --fail-with-body; then
+      err "Service health check failed. Aborting uploads."
+  fi
+  bb_log_info "Service is healthy. Proceeding with uploads."
 fi
-bb_log_info "Service is healthy. Proceeding with uploads."
 
 # Step 2: Validate directory
 if [[ ! -d "$DIR" ]]; then
   err "Error: directory '$DIR' does not exist"
 fi
 
-# Step 3: Find XML files
+# Step 3: Discover files based on content kind
 shopt -s nullglob
-XML_FILES=("$DIR"/*.xml)
+FILES=()
+if [[ "$CONTENT_KIND" == "xml" ]]; then
+  FILES=("$DIR"/*.xml)
+else
+  # stdout.log in: <DIR>/<component>/log/stdout.log
+  FILES=("$DIR"/*/log/stdout.log)
+fi
 shopt -u nullglob
 
-if (( ${#XML_FILES[@]} == 0 )); then
-  err "Error: no .xml files found in directory '$DIR'"
+if (( ${#FILES[@]} == 0 )); then
+  if [[ "$CONTENT_KIND" == "xml" ]]; then
+    err "Error: no .xml files found in directory '$DIR'"
+  else
+    err "Error: no stdout.log files found under '$DIR' (expected: <component>/log/stdout.log)"
+  fi
 fi
 
+# Helper: print an argv array as a shell-escaped command (safe for copy/paste)
+print_cmd() {
+  local -a argv=( "$@" )
+  printf '%q ' "${argv[@]}"
+  printf '\n'
+}
+
 # Step 4: Upload files and track failures
 ANY_FAILED=0
 
-for FILE in "${XML_FILES[@]}"; do
-  # Extract filename without extension for 'typ'
-  BASENAME="$(basename "$FILE" .xml)"
-  bb_log_info "Uploading $FILE (typ=$BASENAME)..."
-
-  if ! curl --max-time 120 --connect-timeout 10 --fail-with-body \
-    -X POST "$UPLOAD_URL" \
-    -F "branch=${BRANCH}" \
-    -F "revision=${REVISION}" \
-    -F "platform=${PLATFORM}" \
-    -F "bbnum=${BBNUM}" \
-    -F "typ=${BASENAME}" \
-    -F "file=@${FILE};type=application/xml"; then
-      bb_log_info "Upload failed for $FILE"
-      ANY_FAILED=1
+for FILE in "${FILES[@]}"; do
+  BASENAME=""
+  MIME_TYPE=""
+
+  if [[ "$CONTENT_KIND" == "xml" ]]; then
+    BASENAME="$(basename "$FILE" .xml)"
+    MIME_TYPE="application/xml"
+  else
+    COMPONENT_DIR="$(dirname "$(dirname "$FILE")")"  # .../<component>
+    BASENAME="$(basename "$COMPONENT_DIR")"
+    MIME_TYPE="text/plain"
+  fi
+
+  bb_log_info "Preparing upload for $FILE (typ=$BASENAME)..."
+
+  CURL_CMD=(
+    curl
+    --max-time 120
+    --connect-timeout 10
+    --fail-with-body
+    -X POST "$UPLOAD_URL"
+    -F "branch=${BRANCH}"
+    -F "revision=${REVISION}"
+    -F "platform=${PLATFORM}"
+    -F "bbnum=${BBNUM}"
+    -F "typ=${BASENAME}"
+    -F "file=@${FILE};type=${MIME_TYPE}"
+  )
+
+  if (( DRY_RUN == 1 )); then
+    bb_log_info "DRY RUN: would execute:"
+    print_cmd "${CURL_CMD[@]}"
+    continue
+  fi
+
+  if ! "${CURL_CMD[@]}"; then
+    bb_log_info "Upload failed for $FILE"
+    ANY_FAILED=1
   else
     bb_log_info "Upload succeeded for $FILE"
   fi
 done
 
 # Step 5: Final result
-if ((ANY_FAILED != 0 )); then
+if (( DRY_RUN == 1 )); then
+  bb_log_info "DRY RUN complete: no network calls were made."
+elif ((ANY_FAILED != 0 )); then
   err "One or more uploads failed."
 else
   bb_log_info "All uploads succeeded."

docker-compose/.env

@@ -2,7 +2,7 @@ TITLE='MariaDB CI'
 TITLE_URL='https://github.com/MariaDB/server'
 BUILDMASTER_URL='https://buildbot.mariadb.org/'
 BUILDMASTER_WG_IP='100.64.100.1'
-JUNIT_COLLECTOR_BASE_URL='http://100.64.100.1:9990'
+MTR_LOG_COLLECTOR_BASE_URL='http://100.64.100.1:9990'
 MQ_ROUTER_URL='ws://127.0.0.1:8080/ws'
 MASTER_PACKAGES_DIR='/mnt/autofs/master_packages'
 MASTER_CREDENTIALS_DIR='/srv/buildbot/master/master-credential-provider'

docker-compose/.env.dev

@@ -2,7 +2,7 @@ TITLE='MariaDB CI (DEV)'
 TITLE_URL='https://github.com/MariaDB/server'
 BUILDMASTER_URL='https://buildbot.dev.mariadb.org/'
 BUILDMASTER_WG_IP='100.64.101.1'
-JUNIT_COLLECTOR_BASE_URL='http://100.64.101.1:9990'
+MTR_LOG_COLLECTOR_BASE_URL='http://100.64.101.1:9990'
 MQ_ROUTER_URL='ws://127.0.0.1:8080/ws'
 MASTER_PACKAGES_DIR='/mnt/autofs/master_dev_packages'
 MASTER_CREDENTIALS_DIR='/srv/buildbot/master/master-credential-provider'