diff --git a/logstash-9.3.yaml b/logstash-9.3.yaml new file mode 100644 index 000000000000..18d76d2e97c5 --- /dev/null +++ b/logstash-9.3.yaml @@ -0,0 +1,524 @@ +# Logstash is a bit of a behemoth because of the way the plugin system works. +# Usually, patched plugins would be dropped on the filesystem as APK's and the +# application could just pick them up. Logstash's plugin system requires the +# plugins to be added by the logstash-plugin cli tool since it uses a generated +# Gemfile. +# +# In order to patch plugins we can do a few things +# +# 1. If the plugin is a default plugin, patch it out of the plugins-metadata.json +# file so the upstream plugin is not installed. Add the plugin apk to the build +# environment and use logstash-plugin to install the .gem file before packaging +# the dist into a tarball (and eventually apk). All of this should be done in +# the main pipeline so the changes are picked up by all subpackages. +# +# 2. If the plugin is not a default plugin, do the same as above, but in a +# subpackage to be installed separately. +package: + name: logstash-9.3 + version: "9.3.0" + epoch: 0 # GHSA-j4pr-3wm6-xx2r + description: Logstash - transport and process your logs, events, or other data + copyright: + - license: Apache-2.0 + resources: + memory: 16Gi + dependencies: + provides: + - ${{package.name}}-base=${{package.full-version}} + - logstash=${{package.full-version}} + runtime: + - bash # some helper scripts use bash and busybox utilities + - busybox + - merged-usrsbin + - openjdk-${{vars.java-version}}-default-jdk + - wolfi-baselayout + +var-transforms: + - from: ${{package.version}} + match: ^(\d+\.\d+)\.\d+$ + replace: $1 + to: major-minor-version + +vars: + java-version: "21" + separately-packaged-plugins: | + logstash-filter-xml logstash-input-beats logstash-input-http logstash-input-tcp logstash-integration-jdbc logstash-input-graphite logstash-input-elastic_serverless_forwarder + +environment: + contents: + packages: + - bash + - ca-certificates-bundle + - coreutils + - curl + - findutils + - glibc-locale-en + - glibc-locales + - gradle + - grep + - jq + - jruby-9.4 + - logstash-filter-xml + - logstash-input-beats + - logstash-input-elastic_serverless_forwarder + - logstash-input-graphite + - logstash-input-http + - logstash-input-tcp + - logstash-integration-jdbc + - logstash-output-opensearch + - net-tools + - openjdk-${{vars.java-version}}-default-jdk + - posix-libc-utils + - sed + - yq + environment: + OSS: "true" + LOGSTASH_SOURCE: "1" + LANG: en_US.UTF-8 + JAVA_HOME: /usr/lib/jvm/default-jvm + # --dev is the default used by logstash-plugin and improves start-up time + JRUBY_OPTS: "--dev -J-Xmx4g" + LS_JAVA_HOME: /usr/lib/jvm/default-jvm + LS_DESTINATION: usr/share/logstash + MAVEN_OPTS: "-Dmaven.repo.local=/var/cache/melange" + GRADLE_USER_HOME: "/var/cache/melange" + GRADLE_OPTS: "-Dorg.gradle.caching=true" + +pipeline: + - uses: git-checkout + with: + repository: https://github.com/elastic/logstash + tag: v${{package.version}} + expected-commit: d6b29147cf2ed1aadaa8dd798f109f344d9b63fb + + - uses: patch + with: + patches: GHSA-3p8m-j85q-pgmj.patch + + - name: Patch sources + runs: | + # Fix GHSA-xc9x-jj77-9p9j + sed -i "s/'date', '>= 3.3.3'/'date', '>= 3.4.1'/" "Gemfile.template" + # Fix GHSA-c2f4-jgmc-q2r5 + sed -i "s/rexml (>= 3\.3\.9)/rexml (>= 3.4.2)/" "Gemfile.jruby-3.1.lock.release" + # Fix GHSA-p543-xpfm-54cp, GHSA-w9pc-fmgc-vxvw and GHSA-wpv5-97wm-hp9c + sed -i "s/rack (>= 3\.1\.16)/rexml (>= 3.1.17)/" "Gemfile.jruby-3.1.lock.release" + echo "gem 'rack', '3.1.18'" >> Gemfile.template + echo "gem 'sinatra', '4.2.0'" >> Gemfile.template + # Fix CVE-2025-27221, GHSA-j4pr-3wm6-xx2r + sed -i "s/'uri', '>= 0.12.3'/'uri', '>= 0.12.5'/" "Gemfile.template" + # Fix CVE-2025-14762 + sed -i 's/aws-sdk-s3 (1.199.1)/aws-sdk-s3 (1.208.0)/' Gemfile.jruby-3.1.lock.release + + for plugin in ${{vars.separately-packaged-plugins}} + do + # Disable the plugin download as we build and package it separately + jq --arg plugin $plugin 'del(.[$plugin])' rakelib/plugins-metadata.json > /tmp/plugins-metadata.json + mv /tmp/plugins-metadata.json rakelib/ + done + + # Disable -Werror which means that warnings are not treated as errors + # during compilation. Required for Java 21. + sed -i '/options.compilerArgs.add("-Werror")/s/^/\/\//' build.gradle + + # A hack-fix to tackle with `Could not find method enabled() for + # arguments [true] on Report xml` error starting after 8.11.x. + sed -i '/enabled/d' logstash-core/build.gradle + + - name: Patch out the windows and mac tarball creation + uses: patch + with: + patches: create_archive_pack.patch + + - name: Install all the default plugins + runs: | + ./gradlew installDefaultGems + + - name: Re-inject default plugins that have been patched + runs: | + for gem in ${{vars.separately-packaged-plugins}} + do + echo "Searching for gem search=$gem-*.gem" + gem_path=$(find /usr/share/jruby -type f -name "$gem-*.gem") + if [ -z "$gem_path" ]; then + echo "Could not find gem gem_path=$gem_path gem=$gem" + exit 1 + fi + echo "Installing gem gem_path=$gem_path" + bin/logstash-plugin install "$gem_path" + done + + - name: Build tarball distribution + runs: | + ./gradlew assembleOssTarDistribution -Pjdk_bundle_os=linux + + - name: Package distribution + runs: | + tarball=$(find . -type f -name "logstash-oss-${{package.version}}-SNAPSHOT-no-jdk.tar.gz") + if [ -z $tarball ]; then + echo "Could not find logstash-oss tarball, nothing to unpack" + exit 1 + fi + + mkdir -p ${{targets.contextdir}}/${LS_DESTINATION} + tar --strip-components 1 \ + -C ${{targets.contextdir}}/${LS_DESTINATION} \ + -xf $tarball + + mkdir -p ${{targets.contextdir}}/usr/bin/$name + for i in ${{targets.contextdir}}/${LS_DESTINATION}/bin/*; do + name=$(basename $i) + ln -sf /${LS_DESTINATION}/bin/$name ${{targets.contextdir}}/usr/bin/$name + done + + - name: Patch CVEs + runs: | + # Hack-fix to reduce CVE count: https://github.com/elastic/logstash/pull/3855 + rm -rf ${{targets.contextdir}}/${LS_DESTINATION}/vendor/bundle/jruby/*/gems/ruby-maven-libs-*/maven-home/lib + +subpackages: + - name: ${{package.name}}-compat + description: Compatibility with the upstream image + dependencies: + replaces: + - ${{package.name}} + - ${{package.name}}-with-output-opensearch + runtime: + - merged-usrsbin + - wolfi-baselayout + pipeline: + - runs: | + src=docker/data/logstash + dst="${{targets.contextdir}}/${LS_DESTINATION}" + mkdir -p "${dst}/config" \ + "${dst}/pipeline" \ + "${{targets.contextdir}}/usr/bin" \ + "${{targets.contextdir}}/usr/local/bin" + + install -m644 "${src}/config/pipelines.yml" "${dst}/config/pipelines.yml" + install -m644 "${src}/config/logstash-oss.yml" "${dst}/config/logstash-oss.yml" + install -m644 "${src}/config/logstash-full.yml" "${dst}/config/logstash-full.yml" + install -m644 "${src}/config/log4j2.file.properties" "${dst}/config/log4j2.file.properties" + install -m644 "${src}/config/log4j2.properties" "${dst}/config/log4j2.properties" + install -m644 "${src}/pipeline/default.conf" "${dst}/pipeline/default.conf" + install -m755 "${src}/bin/docker-entrypoint" "${{targets.contextdir}}/usr/bin/docker-entrypoint" + ln -sf /usr/bin/docker-entrypoint "${{targets.contextdir}}/usr/local/bin/docker-entrypoint" + # test added by a robot (compat) + test: + pipeline: + - runs: | + readlink -v /usr/local/bin/docker-entrypoint + + - name: ${{package.name}}-iamguarded-compat + description: "Logstash for IAMGuarded" + dependencies: + runtime: + - bash # some helper scripts use bash and busybox utilities + - busybox + - coreutils + - findutils + - glibc-locale-en + - glibc-locales + - grep + - net-tools + - openjdk-${{vars.java-version}}-default-jdk + - posix-libc-utils # Required for getent + - sed + - wolfi-baselayout + - yq + pipeline: + - uses: iamguarded/build-compat + with: + package: logstash + version: ${{vars.major-minor-version}} + - runs: | + mkdir -p /opt/iamguarded/logstash + mkdir -p /opt/iamguarded/logstash/pipeline + mkdir -p /opt/iamguarded/scripts/logstash/ + mkdir -p /opt/iamguarded/logstash/config/./ + + src=docker/data/logstash + dst="/opt/iamguarded/logstash" + mkdir -p "${dst}/config" \ + "${dst}/pipeline" + + tarball=$(find . -type f -name "logstash-oss-${{package.version}}-SNAPSHOT-no-jdk.tar.gz") + if [ -z $tarball ]; then + echo "Could not find logstash-oss tarball, nothing to unpack" + exit 1 + fi + + tar --strip-components 1 \ + -C /opt/iamguarded/logstash \ + -xf $tarball + + install -m644 config/jvm.options "${dst}/config/jvm.options" + install -m644 "${src}/config/pipelines.yml" "${dst}/config/pipelines.yml" + install -m644 "${src}/config/logstash-oss.yml" "${dst}/config/logstash-oss.yml" + install -m644 "${src}/config/logstash-full.yml" "${dst}/config/logstash-full.yml" + + /opt/iamguarded/scripts/logstash/postunpack.sh + + # Link binaries and Java + ln -sf /opt/iamguarded/scripts/logstash/entrypoint.sh ${{targets.contextdir}}/entrypoint.sh + ln -sf /opt/iamguarded/scripts/logstash/run.sh ${{targets.contextdir}}/run.sh + - runs: | + # mkdir these empty directories to quiet various startup messages per upstream + mkdir -p /iamguarded/logstash/data + mkdir -p /iamguarded/logstash/pipeline + mkdir -p /iamguarded/logstash/config + - runs: | + # use custom log4j2.properties + cp /opt/iamguarded/logstash/config.default/log4j2.properties /opt/iamguarded/logstash/config.default/log4j2.orig.properties + rm -f /opt/iamguarded/logstash/config.default/log4j2.properties + install -Dm755 log4j2.custom.properties /opt/iamguarded/logstash/config.default/log4j2.properties + # also have a custom logstash.yml + rm /opt/iamguarded/logstash/config.default/logstash.yml + install -Dm755 logstash.iamguarded.yml /opt/iamguarded/logstash/config.default/logstash.yml + - runs: | + find "/opt/iamguarded" \ + -path "/opt/iamguarded/logstash/vendor" -prune -o \ + -exec chmod g+rwX {} \; + find / -perm /6000 -type f -exec chmod a-s {} \; || true + - uses: iamguarded/finalize-compat + with: + package: logstash + version: ${{vars.major-minor-version}} + test: + environment: + contents: + packages: + - bash + - curl + - busybox + - openjdk-${{vars.java-version}} + - sudo + accounts: + groups: + - groupname: logstash + gid: 1001 + users: + - username: logstash + gid: 1001 + uid: 1001 + run-as: 0 + pipeline: + - pipeline: + - uses: iamguarded/test-compat + with: + package: logstash + version: ${{vars.major-minor-version}} + - runs: | + run-script --version + run-script --help + - name: "start daemon on localhost" + uses: test/daemon-check-output + with: + setup: | + chown logstash: -fR /iamguarded/logstash + chown logstash: -fR /opt/iamguarded + start: | + sudo -ulogstash env \ + "LS_JAVA_HOME=/usr/lib/jvm/default-jvm" \ + "PATH=/opt/iamguarded/logstash/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ + /opt/iamguarded/scripts/logstash/entrypoint.sh \ + /opt/iamguarded/scripts/logstash/run.sh + timeout: 180 + expected_output: | + Starting Logstash setup + Starting Logstash + Starting http input listener + post: | + #!/bin/sh -e + sleep 1 + url=http://localhost:8080 + response=$(curl -fsS --connect-timeout 10 --max-time 20 --retry 5 --retry-delay 1 --retry-max-time 40 "$url") || { + echo "curl ${url} failed $?" + exit 1 + } + echo "$response" | grep -q ok || { + echo "response from $url did not contain \"ok\"" + echo "response: $response" + exit 1 + } + echo "$url had expected output: $response" + + - name: ${{package.name}}-env2yaml + description: Merge environment variables into logstash.yml + pipeline: + - runs: | + # Build the Java-based env2yaml tool using the upstream gradle task + ./gradlew :docker:data:logstash:env2yaml:compileJava :docker:data:logstash:env2yaml:copyRuntimeLibs + + # Install compiled classes and dependencies + mkdir -p "${{targets.contextdir}}/usr/share/logstash/env2yaml" + cp -r docker/data/logstash/env2yaml/classes "${{targets.contextdir}}/usr/share/logstash/env2yaml/" + cp -r docker/data/logstash/env2yaml/lib "${{targets.contextdir}}/usr/share/logstash/env2yaml/" + + # Patch and install the wrapper script to use system java instead of bundled JDK + mkdir -p "${{targets.contextdir}}/usr/bin" + sed 's|/usr/share/logstash/jdk/bin/java|java|' \ + docker/data/logstash/env2yaml/env2yaml > "${{targets.contextdir}}/usr/bin/env2yaml" + chmod +x "${{targets.contextdir}}/usr/bin/env2yaml" + dependencies: + runtime: + - merged-usrsbin + - wolfi-baselayout + - openjdk-${{vars.java-version}}-default-jdk + # Due to the way logstash implements their plugin system, this is a full + # logstash package with the opensearch plugin bundled. It is based off the + # same artifacts in the main package, meaning it will pick up the changes to + # the default gems that were made in the main package it just adds the + # opensearch plugin. + test: + pipeline: + - runs: | + # Verify env2yaml files exist + stat /usr/bin/env2yaml + stat /usr/share/logstash/env2yaml/classes + stat /usr/share/logstash/env2yaml/lib + test -x /usr/bin/env2yaml || { echo "ERROR: /usr/bin/env2yaml is not executable"; exit 1; } + - runs: | + # Test that env2yaml works - it modifies the file in place + # env2yaml reads environment variables and injects them into logstash.yml + echo "existing: value" > /tmp/logstash.yml + pipeline_workers=test env2yaml /tmp/logstash.yml + cat /tmp/logstash.yml + # Verify env2yaml injected the variable reference correctly + grep -F 'pipeline.workers: ${pipeline_workers}' /tmp/logstash.yml + # Verify existing configuration was preserved + grep -F 'existing: value' /tmp/logstash.yml + + - name: ${{package.name}}-with-output-opensearch + description: Build logstash with logstash-output-opensearch plugin + dependencies: + replaces: + - ${{package.name}} + runtime: + - bash # some helper scripts use bash and busybox utilities + - busybox + - merged-usrsbin + - openjdk-${{vars.java-version}}-default-jdk + - wolfi-baselayout + pipeline: + - name: Install all the default plugins + runs: | + ./gradlew installDefaultGems + - name: Re-inject default plugins that have been patched + runs: | + for gem in \ + "logstash-output-opensearch"; + do + echo "Searching for gem search=$gem-*.gem" + gem_path=$(find /usr/share/jruby -type f -name "$gem-*.gem") + if [ -z "$gem_path" ]; then + echo "Could not find gem gem_path=$gem_path gem=$gem" + exit 1 + fi + echo "Installing gem gem_path=$gem_path" + bin/logstash-plugin install "$gem_path" + done + - name: Build tarball distribution + runs: | + ./gradlew assembleOssTarDistribution -Pjdk_bundle_os=linux + - name: Package distribution + runs: | + mkdir -p ${{targets.contextdir}}/${LS_DESTINATION} + tar --strip-components 1 \ + -C ${{targets.contextdir}}/${LS_DESTINATION} \ + -xf $(find . -type f -name "logstash-oss-${{package.version}}-SNAPSHOT-no-jdk.tar.gz") + + mkdir -p ${{targets.contextdir}}/usr/bin/$name + for i in ${{targets.contextdir}}/${LS_DESTINATION}/bin/*; do + name=$(basename $i) + ln -sf /${LS_DESTINATION}/bin/$name ${{targets.contextdir}}/usr/bin/$name + done + - name: Patch CVEs + runs: | + # Hack-fix to reduce CVE count: https://github.com/elastic/logstash/pull/3855 + rm -rf ${{targets.contextdir}}/${LS_DESTINATION}/vendor/bundle/jruby/*/gems/ruby-maven-libs-*/maven-home/lib + test: + environment: + contents: + packages: + - openjdk-${{vars.java-version}} + - sudo + accounts: + groups: + - groupname: logstash + gid: 1001 + users: + - username: logstash + gid: 1001 + uid: 1001 + run-as: 0 + environment: + LS_JAVA_HOME: /usr/lib/jvm/default-jvm + pipeline: + - name: Ensure output opensearch plugin was installed + runs: | + logstash-plugin list | grep logstash-output-opensearch + - name: Ensure logstash itself works with basic function + runs: | + # Ensure logstash user can access the files and write to data for tests + chown logstash: -fR /usr/share/logstash/ + + echo "hello" | \ + sudo -ulogstash logstash -e 'input { stdin { } } output { stdout {} }' | \ + grep 'message.*hello' + - uses: test/tw/ldd-check + +test: + environment: + contents: + packages: + - openjdk-${{vars.java-version}}-default-jdk + - sudo + accounts: + groups: + - groupname: logstash + gid: 1001 + users: + - username: logstash + gid: 1001 + uid: 1001 + run-as: 0 + environment: + LS_JAVA_HOME: /usr/lib/jvm/default-jvm + pipeline: + - uses: test/tw/ldd-check + - name: Ensure default plugins were actually installed + runs: | + for plugin in ${{vars.separately-packaged-plugins}} + do + logstash-plugin list | grep $plugin + done + logstash --version + logstash.lib.sh --version + logstash.lib.sh --help + pqcheck --help + ruby --version + ruby --help + system-install --help + - name: Ensure logstash itself works with basic function + runs: | + # Ensure logstash user can access the files and write to data for tests + chown logstash: -fR /usr/share/logstash/ + + echo "hello" | \ + sudo -ulogstash logstash -e 'input { stdin { } } output { stdout {} }' | \ + grep 'message.*hello' + - name: Test logstash-filter-xml + runs: | + # Copy the file to make it readable by logstash user + cp $PWD/test/logstash-filter-xml/logstash.conf /tmp/logstash.conf + sudo -ulogstash logstash -f /tmp/logstash.conf + +update: + enabled: true + github: + identifier: elastic/logstash + strip-prefix: v + use-tag: true + tag-filter: v9.3. diff --git a/logstash-9.3/GHSA-3p8m-j85q-pgmj.patch b/logstash-9.3/GHSA-3p8m-j85q-pgmj.patch new file mode 100644 index 000000000000..efb6cf6c0b76 --- /dev/null +++ b/logstash-9.3/GHSA-3p8m-j85q-pgmj.patch @@ -0,0 +1,12 @@ +diff --git a/logstash-core/build.gradle b/logstash-core/build.gradle +index 14dd90e81..f41d2be77 100644 +--- a/logstash-core/build.gradle ++++ b/logstash-core/build.gradle +@@ -245,6 +245,7 @@ dependencies { + exclude group: 'com.google.guava', module: 'guava' + } + implementation 'org.javassist:javassist:3.30.2-GA' ++ implementation 'io.netty:netty-codec:4.1.125.Final' + testImplementation "org.apache.logging.log4j:log4j-core:${log4jVersion}:tests" + testImplementation 'org.hamcrest:hamcrest:2.2' + testImplementation 'org.hamcrest:hamcrest-library:2.2' diff --git a/logstash-9.3/create_archive_pack.patch b/logstash-9.3/create_archive_pack.patch new file mode 100644 index 000000000000..e26ea003e8e9 --- /dev/null +++ b/logstash-9.3/create_archive_pack.patch @@ -0,0 +1,26 @@ +diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake +index 752c83ad3..d37c5be71 100644 +--- a/rakelib/artifacts.rake ++++ b/rakelib/artifacts.rake +@@ -173,19 +173,12 @@ namespace "artifact" do + + desc "Build all (jdk bundled and not) tar.gz and zip of default logstash plugins with all dependencies" + task "archives" => ["prepare", "generate_build_metadata"] do +- #with bundled JDKs +- license_details = ['ELASTIC-LICENSE'] +- @bundles_jdk = true +- if ARCH == "x86_64" +- create_archive_pack(license_details, "x86_64", "linux", "windows", "darwin") +- else +- create_archive_pack(license_details, "arm64", "linux", "darwin") +- end ++ + #without JDK + safe_system("./gradlew bootstrap") #force the build of Logstash jars + @bundles_jdk = false ++ license_details = ['APACHE-LICENSE-2.0', "-oss", oss_exclude_paths] + build_tar(*license_details, platform: '-no-jdk') +- build_zip(*license_details, platform: '-no-jdk') + end + + desc "Build jdk bundled tar.gz of default logstash plugins with all dependencies for docker" diff --git a/logstash-9.3/log4j2.custom.properties b/logstash-9.3/log4j2.custom.properties new file mode 100644 index 000000000000..663a0158034f --- /dev/null +++ b/logstash-9.3/log4j2.custom.properties @@ -0,0 +1,16 @@ +status = error +name = LogstashPropertiesConfig + +appender.console.type = Console +appender.console.name = plain_console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c]%notEmpty{[%X{pipeline.id}]}%notEmpty{[%X{plugin.id}]} %m%n + +appender.json_console.type = Console +appender.json_console.name = json_console +appender.json_console.layout.type = JSONLayout +appender.json_console.layout.compact = true +appender.json_console.layout.eventEol = true + +rootLogger.level = ${sys:ls.log.level} +rootLogger.appenderRef.console.ref = ${sys:ls.log.format}_console diff --git a/logstash-9.3/logstash.iamguarded.yml b/logstash-9.3/logstash.iamguarded.yml new file mode 100644 index 000000000000..dd86531aa761 --- /dev/null +++ b/logstash-9.3/logstash.iamguarded.yml @@ -0,0 +1,397 @@ +# Settings file in YAML +# +# Settings can be specified either in hierarchical form, e.g.: +# +# pipeline: +# batch: +# size: 125 +# delay: 5 +# +# Or as flat keys: +# +# pipeline.batch.size: 125 +# pipeline.batch.delay: 5 +# +# ------------ Node identity ------------ +# +# Use a descriptive name for the node: +# +# node.name: test +# +# If omitted the node name will default to the machine's host name +# +# ------------ Data path ------------------ +# +# Which directory should be used by logstash and its plugins +# for any persistent needs. Defaults to LOGSTASH_HOME/data +# +# path.data: +# +# ------------ Pipeline Settings -------------- +# +# The ID of the pipeline. +# +# pipeline.id: main +# +# Set the number of workers that will, in parallel, execute the filters+outputs +# stage of the pipeline. +# +# This defaults to the number of the host's CPU cores. +# +# pipeline.workers: 2 +# +# How many events to retrieve from inputs before sending to filters+workers +# +# pipeline.batch.size: 125 +# +# How long to wait in milliseconds while polling for the next event +# before dispatching an undersized batch to filters+outputs +# +# pipeline.batch.delay: 50 +# +# Force Logstash to exit during shutdown even if there are still inflight +# events in memory. By default, logstash will refuse to quit until all +# received events have been pushed to the outputs. +# +# WARNING: Enabling this can lead to data loss during shutdown +# +# pipeline.unsafe_shutdown: false +# +# Set the pipeline event ordering. Options are "auto" (the default), "true" or "false". +# "auto" automatically enables ordering if the 'pipeline.workers' setting +# is also set to '1', and disables otherwise. +# "true" enforces ordering on the pipeline and prevent logstash from starting +# if there are multiple workers. +# "false" disables any extra processing necessary for preserving ordering. +# +# pipeline.ordered: auto +# +# Sets the pipeline's default value for `ecs_compatibility`, a setting that is +# available to plugins that implement an ECS Compatibility mode for use with +# the Elastic Common Schema. +# Possible values are: +# - disabled +# - v1 +# - v8 (default) +# Pipelines defined before Logstash 8 operated without ECS in mind. To ensure a +# migrated pipeline continues to operate as it did before your upgrade, opt-OUT +# of ECS for the individual pipeline in its `pipelines.yml` definition. Setting +# it here will set the default for _all_ pipelines, including new ones. +# +# pipeline.ecs_compatibility: v8 +# +# ------------ Pipeline Configuration Settings -------------- +# +# Where to fetch the pipeline configuration for the main pipeline +# +# path.config: +# +# Pipeline configuration string for the main pipeline +# +# config.string: +# +# At startup, test if the configuration is valid and exit (dry run) +# +# config.test_and_exit: false +# +# Periodically check if the configuration has changed and reload the pipeline +# This can also be triggered manually through the SIGHUP signal +# +# config.reload.automatic: false +# +# How often to check if the pipeline configuration has changed (in seconds) +# Note that the unit value (s) is required. Values without a qualifier (e.g. 60) +# are treated as nanoseconds. +# Setting the interval this way is not recommended and might change in later versions. +# +# config.reload.interval: 3s +# +# Show fully compiled configuration as debug log message +# NOTE: --log.level must be 'debug' +# +# config.debug: false +# +# When enabled, process escaped characters such as \n and \" in strings in the +# pipeline configuration files. +# +# config.support_escapes: false +# +# ------------ API Settings ------------- +# Define settings related to the HTTP API here. +# +# The HTTP API is enabled by default. It can be disabled, but features that rely +# on it will not work as intended. +# +# api.enabled: true +# +# By default, the HTTP API is not secured and is therefore bound to only the +# host's loopback interface, ensuring that it is not accessible to the rest of +# the network. +# When secured with SSL and Basic Auth, the API is bound to _all_ interfaces +# unless configured otherwise. +# +# api.http.host: 127.0.0.1 +# +# The HTTP API web server will listen on an available port from the given range. +# Values can be specified as a single port (e.g., `9600`), or an inclusive range +# of ports (e.g., `9600-9700`). +# +# api.http.port: 9600-9700 +# +# The HTTP API includes a customizable "environment" value in its response, +# which can be configured here. +# +# api.environment: "production" +# +# The HTTP API can be secured with SSL (TLS). To do so, you will need to provide +# the path to a password-protected keystore in p12 or jks format, along with credentials. +# +# api.ssl.enabled: false +# api.ssl.keystore.path: /path/to/keystore.jks +# api.ssl.keystore.password: "y0uRp4$$w0rD" +# +# The availability of SSL/TLS protocols depends on the JVM version. Certain protocols are +# disabled by default and need to be enabled manually by changing `jdk.tls.disabledAlgorithms` +# in the $JDK_HOME/conf/security/java.security configuration file. +# +# api.ssl.supported_protocols: [TLSv1.2,TLSv1.3] +# +# The HTTP API can be configured to require authentication. Acceptable values are +# - `none`: no auth is required (default) +# - `basic`: clients must authenticate with HTTP Basic auth, as configured +# with `api.auth.basic.*` options below +# api.auth.type: none +# +# When configured with `api.auth.type` `basic`, you must provide the credentials +# that requests will be validated against. Usage of Environment or Keystore +# variable replacements is encouraged (such as the value `"${HTTP_PASS}"`, which +# resolves to the value stored in the keystore's `HTTP_PASS` variable if present +# or the same variable from the environment) +# +# api.auth.basic.username: "logstash-user" +# api.auth.basic.password: "s3cUreP4$$w0rD" +# +# When setting `api.auth.basic.password`, the password should meet +# the default password policy requirements. +# The default password policy requires non-empty minimum 8 char string that +# includes a digit, upper case letter and lower case letter. +# Policy mode sets Logstash to WARN or ERROR when HTTP authentication password doesn't +# meet the password policy requirements. +# The default is WARN. Setting to ERROR enforces stronger passwords (recommended). +# +# api.auth.basic.password_policy.mode: WARN +# +# ------------ Module Settings --------------- +# Define modules here. Modules definitions must be defined as an array. +# The simple way to see this is to prepend each `name` with a `-`, and keep +# all associated variables under the `name` they are associated with, and +# above the next, like this: +# +# modules: +# - name: MODULE_NAME +# var.PLUGINTYPE1.PLUGINNAME1.KEY1: VALUE +# var.PLUGINTYPE1.PLUGINNAME1.KEY2: VALUE +# var.PLUGINTYPE2.PLUGINNAME1.KEY1: VALUE +# var.PLUGINTYPE3.PLUGINNAME3.KEY1: VALUE +# +# Module variable names must be in the format of +# +# var.PLUGIN_TYPE.PLUGIN_NAME.KEY +# +# modules: +# +# ------------ Cloud Settings --------------- +# Define Elastic Cloud settings here. +# Format of cloud.id is a base64 value e.g. dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy +# and it may have an label prefix e.g. staging:dXMtZ... +# This will overwrite 'var.elasticsearch.hosts' and 'var.kibana.host' +# cloud.id: +# +# Format of cloud.auth is: : +# This is optional +# If supplied this will overwrite 'var.elasticsearch.username' and 'var.elasticsearch.password' +# If supplied this will overwrite 'var.kibana.username' and 'var.kibana.password' +# cloud.auth: elastic: +# +# ------------ Queuing Settings -------------- +# +# Internal queuing model, "memory" for legacy in-memory based queuing and +# "persisted" for disk-based acked queueing. Defaults is memory +# +# queue.type: memory +# +# If `queue.type: persisted`, the directory path where the pipeline data files will be stored. +# Each pipeline will group its PQ files in a subdirectory matching its `pipeline.id`. +# Default is path.data/queue. +# +# path.queue: +# +# If using queue.type: persisted, the page data files size. The queue data consists of +# append-only data files separated into pages. Default is 64mb +# +# queue.page_capacity: 64mb +# +# If using queue.type: persisted, the maximum number of unread events in the queue. +# Default is 0 (unlimited) +# +# queue.max_events: 0 +# +# If using queue.type: persisted, the total capacity of the queue in number of bytes. +# If you would like more unacked events to be buffered in Logstash, you can increase the +# capacity using this setting. Please make sure your disk drive has capacity greater than +# the size specified here. If both max_bytes and max_events are specified, Logstash will pick +# whichever criteria is reached first +# Default is 1024mb or 1gb +# +# queue.max_bytes: 1024mb +# +# If using queue.type: persisted, the maximum number of acked events before forcing a checkpoint +# Default is 1024, 0 for unlimited +# +# queue.checkpoint.acks: 1024 +# +# If using queue.type: persisted, the maximum number of written events before forcing a checkpoint +# Default is 1024, 0 for unlimited +# +# queue.checkpoint.writes: 1024 +# +# If using queue.type: persisted, the interval in milliseconds when a checkpoint is forced on the head page +# Default is 1000, 0 for no periodic checkpoint. +# +# queue.checkpoint.interval: 1000 +# +# ------------ Dead-Letter Queue Settings -------------- +# Flag to turn on dead-letter queue. +# +# dead_letter_queue.enable: false +# +# If using dead_letter_queue.enable: true, the maximum size of each dead letter queue. Entries +# will be dropped if they would increase the size of the dead letter queue beyond this setting. +# Default is 1024mb +# dead_letter_queue.max_bytes: 1024mb +# +# If using dead_letter_queue.enable: true, the interval in milliseconds where if no further events eligible for the DLQ +# have been created, a dead letter queue file will be written. A low value here will mean that more, smaller, queue files +# may be written, while a larger value will introduce more latency between items being "written" to the dead letter queue, and +# being available to be read by the dead_letter_queue input when items are written infrequently. +# Default is 5000. +# +# dead_letter_queue.flush_interval: 5000 +# +# If using dead_letter_queue.enable: true, controls which entries should be dropped to avoid exceeding the size limit. +# Set the value to `drop_newer` (default) to stop accepting new events that would push the DLQ size over the limit. +# Set the value to `drop_older` to remove queue pages containing the oldest events to make space for new ones. +# +# dead_letter_queue.storage_policy: drop_newer +# +# If using dead_letter_queue.enable: true, the interval that events have to be considered valid. After the interval has +# expired the events could be automatically deleted from the DLQ. +# The interval could be expressed in days, hours, minutes or seconds, using as postfix notation like 5d, +# to represent a five days interval. +# The available units are respectively d, h, m, s for day, hours, minutes and seconds. +# If not specified then the DLQ doesn't use any age policy for cleaning events. +# +# dead_letter_queue.retain.age: 1d +# +# If using dead_letter_queue.enable: true, the directory path where the data files will be stored. +# Default is path.data/dead_letter_queue +# +# path.dead_letter_queue: +# +# ------------ Debugging Settings -------------- +# +# Options for log.level: +# * fatal +# * error +# * warn +# * info (default) +# * debug +# * trace +# log.level: info +# +# Options for log.format: +# * plain (default) +# * json +# +# log.format: plain +# log.format.json.fix_duplicate_message_fields: false +# +# path.logs: +# +# ------------ Other Settings -------------- +# +# Allow or block running Logstash as superuser (default: true) +# allow_superuser: false +# +# Where to find custom plugins +# path.plugins: [] +# +# Flag to output log lines of each pipeline in its separate log file. Each log filename contains the pipeline.name +# Default is false +# pipeline.separate_logs: false +# +# Determine where to allocate memory buffers, for plugins that leverage them. +# Default to direct, optionally can be switched to heap to select Java heap space. +# pipeline.buffer.type: direct +# +# ------------ X-Pack Settings (not applicable for OSS build)-------------- +# +# X-Pack Monitoring +# https://www.elastic.co/guide/en/logstash/current/monitoring-logstash.html +#xpack.monitoring.enabled: false +#xpack.monitoring.elasticsearch.username: logstash_system +#xpack.monitoring.elasticsearch.password: password +#xpack.monitoring.elasticsearch.proxy: ["http://proxy:port"] +#xpack.monitoring.elasticsearch.hosts: ["https://es1:9200", "https://es2:9200"] +# an alternative to hosts + username/password settings is to use cloud_id/cloud_auth +#xpack.monitoring.elasticsearch.cloud_id: monitoring_cluster_id:xxxxxxxxxx +#xpack.monitoring.elasticsearch.cloud_auth: logstash_system:password +# another authentication alternative is to use an Elasticsearch API key +#xpack.monitoring.elasticsearch.api_key: "id:api_key" +#xpack.monitoring.elasticsearch.ssl.certificate_authority: "/path/to/ca.crt" +#xpack.monitoring.elasticsearch.ssl.ca_trusted_fingerprint: xxxxxxxxxx +#xpack.monitoring.elasticsearch.ssl.truststore.path: path/to/file +#xpack.monitoring.elasticsearch.ssl.truststore.password: password +# use either keystore.path/keystore.password or certificate/key configurations +#xpack.monitoring.elasticsearch.ssl.keystore.path: /path/to/file +#xpack.monitoring.elasticsearch.ssl.keystore.password: password +#xpack.monitoring.elasticsearch.ssl.certificate: /path/to/file +#xpack.monitoring.elasticsearch.ssl.key: /path/to/key +#xpack.monitoring.elasticsearch.ssl.verification_mode: full +#xpack.monitoring.elasticsearch.ssl.cipher_suites: [] +#xpack.monitoring.elasticsearch.sniffing: false +#xpack.monitoring.collection.interval: 10s +#xpack.monitoring.collection.pipeline.details.enabled: true +# +# X-Pack Management +# https://www.elastic.co/guide/en/logstash/current/logstash-centralized-pipeline-management.html +#xpack.management.enabled: false +#xpack.management.pipeline.id: ["main", "apache_logs"] +#xpack.management.elasticsearch.username: logstash_admin_user +#xpack.management.elasticsearch.password: password +#xpack.management.elasticsearch.proxy: ["http://proxy:port"] +#xpack.management.elasticsearch.hosts: ["https://es1:9200", "https://es2:9200"] +# an alternative to hosts + username/password settings is to use cloud_id/cloud_auth +#xpack.management.elasticsearch.cloud_id: management_cluster_id:xxxxxxxxxx +#xpack.management.elasticsearch.cloud_auth: logstash_admin_user:password +# another authentication alternative is to use an Elasticsearch API key +#xpack.management.elasticsearch.api_key: "id:api_key" +#xpack.management.elasticsearch.ssl.ca_trusted_fingerprint: xxxxxxxxxx +#xpack.management.elasticsearch.ssl.certificate_authority: "/path/to/ca.crt" +#xpack.management.elasticsearch.ssl.truststore.path: /path/to/file +#xpack.management.elasticsearch.ssl.truststore.password: password +# use either keystore.path/keystore.password or certificate/key configurations +#xpack.management.elasticsearch.ssl.keystore.path: /path/to/file +#xpack.management.elasticsearch.ssl.keystore.password: password +#xpack.management.elasticsearch.ssl.certificate: /path/to/file +#xpack.management.elasticsearch.ssl.key: /path/to/certificate_key_file +#xpack.management.elasticsearch.ssl.cipher_suites: [] +#xpack.management.elasticsearch.ssl.verification_mode: full +#xpack.management.elasticsearch.sniffing: false +#xpack.management.logstash.poll_interval: 5s +# +# X-Pack GeoIP Database Management +# https://www.elastic.co/guide/en/logstash/current/plugins-filters-geoip.html#plugins-filters-geoip-manage_update +#xpack.geoip.downloader.enabled: true +#xpack.geoip.downloader.endpoint: "https://geoip.elastic.co/v1/database" +path.data: /iamguarded/logstash/data diff --git a/logstash-9.3/test/logstash-filter-xml/logstash.conf b/logstash-9.3/test/logstash-filter-xml/logstash.conf new file mode 100644 index 000000000000..84f6df91828b --- /dev/null +++ b/logstash-9.3/test/logstash-filter-xml/logstash.conf @@ -0,0 +1,30 @@ +input { + file { + # Needs to be an absolute path or logstash will ignore + path => "/home/build/test/logstash-filter-xml/test.xml" + mode => read + start_position => "beginning" + sincedb_path => "/dev/null" + codec => multiline { + pattern => "^" + negate => "true" + what => "previous" + } + exit_after_read => true + } +} + +filter { + xml { + source => "message" + store_xml => "false" + xpath => [ + "/root/item/name/text()", "item_name", + "/root/item/price/text()", "item_price" + ] + } +} + +output { + stdout { codec => rubydebug } +} diff --git a/logstash-9.3/test/logstash-filter-xml/test.xml b/logstash-9.3/test/logstash-filter-xml/test.xml new file mode 100644 index 000000000000..675aca7585df --- /dev/null +++ b/logstash-9.3/test/logstash-filter-xml/test.xml @@ -0,0 +1,11 @@ + + + Item 1 + 10.99 + + + Item 2 + 20.49 + + +