fix: ensure we build the agent registration plugin (#322)

* fix: ensure we build the agent registration plugin
* test: add missing helper functions
* fix: add git and other missing parts to UBI container image

Signed-off-by: Patrick Stephens <pat@telemetryforge.io>

Author: patrick-stephens

Dockerfile.ubi

@@ -1,8 +1,11 @@
+# syntax=docker/dockerfile:1
+
 FROM registry.access.redhat.com/ubi9:9.7 AS base-builder
 
 # hadolint ignore=DL3041
 RUN dnf update -y && \
-    dnf install -y cmake3 diffutils gcc gcc-c++ git m4 make openssl-devel patch systemd-devel tar unzip libyaml-devel cyrus-sasl-devel zlib-devel pcre2-devel && \
+    dnf install -y cmake3 diffutils gcc gcc-c++ git m4 make openssl-devel patch systemd-devel tar unzip \
+    libyaml-devel cyrus-sasl-devel zlib-devel pcre2-devel libzstd-devel && \
     dnf clean all && rm -rf /var/cache/dnf
 
 # Provide the dependencies for record accessor built from source
@@ -28,6 +31,7 @@ RUN curl -LO https://www.libssh2.org/download/libssh2-${LIBSSH2_VER}.tar.gz && \
 WORKDIR /tmp/libssh2-${LIBSSH2_VER}
 RUN ./configure --with-openssl --prefix=/usr/local --disable-examples-build && \
     make -j "$(getconf _NPROCESSORS_ONLN)" && make install && \
+    echo "/usr/local/lib" > /etc/ld.so.conf.d/libssh2.conf && \
     ldconfig
 
 # Build libgit2 from source
@@ -57,7 +61,7 @@ ARG TELEMETRY_FORGE_AGENT_VERSION=26.4.8
 ENV TELEMETRY_FORGE_AGENT_VERSION=${TELEMETRY_FORGE_AGENT_VERSION}
 
 # Build metadata
-ARG TELEMETRY_FORGE_AGENT_DISTRO=ubi/9.5
+ARG TELEMETRY_FORGE_AGENT_DISTRO=ubi/9.7
 ENV TELEMETRY_FORGE_AGENT_DISTRO=$TELEMETRY_FORGE_AGENT_DISTRO
 
 ARG TELEMETRY_FORGE_AGENT_PACKAGE_TYPE=CONTAINER
@@ -95,6 +99,7 @@ RUN cmake -DCMAKE_BUILD_TYPE=Release \
     -DTELEMETRY_FORGE_AGENT_VERSION="$TELEMETRY_FORGE_AGENT_VERSION" \
     -DFLB_LOG_NO_CONTROL_CHARS=On \
     -DFLB_CORO_STACK_SIZE=131072 \
+    -DFLB_PREFER_SYSTEM_LIB_ZSTD=On \
     ${CMAKE_EXTRA_ARGS} \
     ..
 
@@ -111,7 +116,7 @@ RUN [ -n "$MAKE_EXTRA_ARGS" ] && make ${MAKE_EXTRA_ARGS} || make -j "$(getconf _
 FROM registry.access.redhat.com/ubi9/ubi-minimal:9.7 AS production
 
 # hadolint ignore=DL3041
-RUN microdnf update -y && microdnf install -y openssl libyaml cyrus-sasl && microdnf clean all
+RUN microdnf update -y && microdnf install -y openssl libyaml cyrus-sasl zstd && microdnf clean all
 
 EXPOSE 2020
 
@@ -127,7 +132,7 @@ COPY --from=builder /fluent-bit /fluent-bit
 # Copy libgit2 and libssh2 runtime libraries from builder
 COPY --from=builder /usr/local/lib64/libgit2.so.*.*.* /usr/local/lib64/
 COPY --from=builder /usr/local/lib/libssh2.so.*.*.* /usr/local/lib/
-# Set up ldconfig and symlinks for the copied libraries
+# Set up ldconfig and symlinks for the libraries
 RUN echo "/usr/local/lib64" > /etc/ld.so.conf.d/local-libs.conf && \
     echo "/usr/local/lib" >> /etc/ld.so.conf.d/local-libs.conf && \
     ldconfig
@@ -170,8 +175,9 @@ FROM production AS test
 USER root
 
 # We need find and ps/pkill for BATS
+# We need git for the git_config plugin tests
 # hadolint ignore=DL3041
-RUN microdnf update -y && microdnf install -y findutils procps && microdnf clean all
+RUN microdnf update -y && microdnf install -y findutils procps git && microdnf clean all
 
 COPY --from=bats /opt/bats /opt/bats
 RUN /opt/bats/install.sh /usr/local

source/cmake/plugins_options.cmake

@@ -11,11 +11,11 @@ option(FLB_MINIMAL "Enable minimal build configuration" No)
 
 # Custom Plugins
 # ===============
-DEFINE_OPTION(FLB_CUSTOM_TELEMETRY_FORGE "Enable Telemetry Forge custom plugin" ON)
+DEFINE_OPTION(FLB_CUSTOM_TELEMETRYFORGE "Enable Telemetry Forge custom plugin" ON)
 
 # Inputs (sources, data collectors)
 # =================================
-DEFINE_OPTION(FLB_IN_TELEMETRY_FORGE "Enable Telemetry Forge input plugin" ON)
+DEFINE_OPTION(FLB_IN_TELEMETRYFORGE "Enable Telemetry Forge input plugin" ON)
 # =================================
 DEFINE_OPTION(FLB_IN_BLOB                     "Enable Blob input plugin"                     ON)
 DEFINE_OPTION(FLB_IN_CALYPTIA_FLEET           "Enable Calyptia Fleet input plugin"           OFF)

testing/Dockerfile.bats

@@ -8,6 +8,26 @@ FROM ${BASE_BUILDER} AS test
 COPY --from=bats /opt/bats /opt/bats
 RUN /opt/bats/install.sh /usr/local
 
+# Install git for testing the git config plugin
+RUN if ! command -v git &>/dev/null; then \
+        if command -v yum &>/dev/null; then \
+            yum install -y git; \
+        elif command -v microdnf &>/dev/null; then \
+            microdnf install -y git; \
+        elif command -v dnf &>/dev/null; then \
+            dnf install -y git; \
+        elif command -v apk &>/dev/null; then \
+            apk add --no-cache git; \
+        elif command -v zypper &>/dev/null; then \
+            zypper install -y git; \
+        elif command -v apt-get &>/dev/null; then \
+            apt-get update && apt-get install -y git; \
+        else \
+            echo "ERROR: unable to install git"; \
+            exit 1; \
+        fi \
+    fi
+
 COPY testing/ /testing/
 
 # Put packages to install here

testing/bats/helpers/test-helpers.bash

@@ -228,3 +228,86 @@ function helmTeardown() {
     fi
 	export DETIK_CLIENT_NAMESPACE=''
 }
+
+# Starts fluent-bit in the background, redirecting output to a temp file
+# Sets FB_PID and FB_LOG as global variables
+function startFluentBit() {
+    FB_LOG=$(mktemp)
+    "$FLUENT_BIT_BINARY" "$@" > "$FB_LOG" 2>&1 &
+    FB_PID=$!
+	echo "Started $FLUENT_BIT_BINARY with PID $FB_PID and log file $FB_LOG: $*"
+}
+
+# After a successful match, call this to record the current log position.
+# Future calls to waitForOutput can then only check for new output since this point
+function markLogPosition() {
+	if [[ -z "${FB_LOG:-}" ]]; then
+		fail "FB_LOG not set, cannot mark log position"
+	fi
+	FB_LOG_OFFSET=$(wc -c < "$FB_LOG")
+}
+
+function resetLogPosition() {
+	unset FB_LOG_OFFSET
+}
+
+# Waits for a specific pattern to appear in the log
+# Usage: waitForOutput "pattern" [timeout_seconds]
+# Default timeout is 30 seconds
+function waitForOutputFromOffset() {
+    local pattern="$1"
+    local timeout="${2:-30}"
+    local offset="${3:-0}"
+    local elapsed=0
+    local interval=1
+
+	if [[ -z "${FB_LOG:-}" ]]; then
+		fail "FB_LOG not set, cannot wait for output"
+	fi
+
+	if [[ -z "${FB_PID:-}" ]]; then
+		fail "FB_PID not set, cannot wait for output"
+	fi
+
+    while (( elapsed < timeout )); do
+        if tail -c +"$((offset + 1))" "$FB_LOG" 2>/dev/null | grep -qE "$pattern"; then
+            return 0
+        fi
+
+        if ! kill -0 "$FB_PID" 2>/dev/null; then
+            cat "$FB_LOG"
+            fail "$FLUENT_BIT_BINARY exited unexpectedly."
+            return 1
+        fi
+
+        sleep "$interval"
+        elapsed=$(( elapsed + interval ))
+    done
+
+    echo "Log contents (from offset ${offset}):"
+    tail -c +"$((offset + 1))" "$FB_LOG"
+    fail "Timed out after ${timeout}s waiting for new pattern: $pattern"
+	return 1
+}
+
+# Wait for a specific pattern to appear in the log since the start of the log (i.e. not just new output since the last marked position)
+function waitForOutput() {
+	waitForOutputFromOffset "$1" "$2" 0
+}
+
+# Wait for a specific pattern to appear in the log since the last marked position
+function waitForNewOutput() {
+	waitForOutputFromOffset "$1" "$2" "${FB_LOG_OFFSET:-0}"
+}
+
+# Cleanup function to kill fluent-bit and remove temp files
+function stopFluentBit() {
+    if [[ -n "${FB_PID:-}" ]] && kill -0 "$FB_PID" 2>/dev/null; then
+        kill -9 "$FB_PID"
+        wait "$FB_PID" 2>/dev/null || true
+    fi
+    if [[ -n "${FB_LOG:-}" ]]; then
+        rm -f "$FB_LOG"
+    fi
+	resetLogPosition
+}

testing/bats/tests/functional/plugins/git_config/resources/fluent-bit-no-exit.yaml

@@ -0,0 +1,15 @@
+service:
+  flush: 1
+  grace: 1
+  log_level: info
+
+pipeline:
+  inputs:
+    - name: dummy
+      dummy: '{"message": "Dummy initial message"}'
+      alias: input_dummy
+
+  outputs:
+    - name: stdout
+      match: '*'
+      alias: output_stdout

testing/bats/tests/functional/plugins/git_config/resources/fluent-bit.yaml

@@ -0,0 +1,20 @@
+service:
+  flush: 1
+  grace: 1
+  log_level: info
+
+pipeline:
+  inputs:
+    - name: dummy
+      dummy: '{"message": "Switched to updated configuration", "sha": "${GIT_SHA}"}'
+      alias: input_dummy
+
+  outputs:
+    - name: stdout
+      match: '*'
+      alias: output_stdout
+    - name: exit
+      match: '*'
+      # Exit after processing 2 records to ensure the test completes in a reasonable time frame
+      record_count: 2
+      alias: output_exit

testing/bats/tests/functional/plugins/git_config/resources/initial-fluent-bit-no-exit.yaml

@@ -0,0 +1,17 @@
+service:
+  flush: 1
+  grace: 1
+  log_level: debug
+
+customs:
+  - name: git_config
+    # We use a local git repository for testing to keep things simple and consistent.
+    repo: file://${GIT_REPO_PATH}
+    # We may need to adjust this per-PR appropriately to ensure we are testing the correct branch/PR
+    ref: ${GIT_SHA}
+    path: fluent-bit-no-exit.yaml
+    # Do not poll too fast as we still need to apply the commit between polls
+    poll_interval: 10
+    # Use a different config directory to prevent conflicts with other tests
+    # See https://github.com/telemetryforge/agent/issues/274
+    config_dir: ${BATS_TEST_TMPDIR}/tmp/fluent-bit

testing/bats/tests/functional/plugins/git_config/resources/initial-fluent-bit.yaml

@@ -0,0 +1,30 @@
+service:
+  flush: 1
+  grace: 1
+  log_level: debug
+
+customs:
+  - name: git_config
+    # We use a local git repository for testing to keep things simple and consistent.
+    repo: file://${GIT_REPO_PATH}
+    # We may need to adjust this per-PR appropriately to ensure we are testing the correct branch/PR
+    ref: ${GIT_SHA}
+    path: fluent-bit.yaml
+    # Do not poll too fast as we still need to apply the commit between polls
+    poll_interval: 10
+    # Use a different config directory to prevent conflicts with other tests
+    # See https://github.com/telemetryforge/agent/issues/274
+    config_dir: ${BATS_TEST_TMPDIR}/tmp/fluent-bit
+
+# Fall back to ensure we exit after 60 seconds in case the plugin doesn't work as expected to prevent hanging the test indefinitely
+# See https://github.com/telemetryforge/agent/issues/272 to remove this later once implemented.
+pipeline:
+  inputs:
+    - name: dummy
+      dummy: '{"message": "Ignored"}'
+      alias: input_dummy_ignored
+  outputs:
+    - name: exit
+      match: '*'
+      time_count: 60
+      alias: output_exit_fallback