Support building ARM archive files locally using the Gradle build.

release/archives/README.md

@@ -3,6 +3,7 @@
 **Supported Platform**
 
 * Linux - x64
+* Linux - arm64
 
 ### General Instructions
 
@@ -11,21 +12,18 @@
 To build all archives, check out the corresponding branch for the version and follow below steps
 
 1. Building an archive depends on the platform on which it will be executed/run
-2. For building archives for all supported platforms, 
-   * from root project run  `./gradlew :release:archives:buildArchives -Prelease`
-   * or from current project `gradle buildArchives -Prelease`
+2. For building archives for all supported platforms, from root project run  `./gradlew :release:archives:buildArchives`
 3. Successful build will generate archives in `release/archives/<platform>/build/distributions/` directory
 4. Generated archives includes a script file which can be used to execute the data prepper using
 
 ```
-tar -xzf opendistroforelasticsearch-data-prepper-jdk-v<VERSION>-<PLATFORM>-<ARCHITECTURE>.tar.gz
-./opendistroforelasticsearch-data-prepper-jdk-v<VERSION>-<PLATFORM>-<ARCHITECTURE>/data-prepper-tar-install.sh <CONFIG FILE LOCATION>
+tar -xzf opensearch-data-prepper-jdk-v<VERSION>-<PLATFORM>-<ARCHITECTURE>.tar.gz
+./opensearch-data-prepper-jdk-v<VERSION>-<PLATFORM>-<ARCHITECTURE>/bin/data-prepper <CONFIG FILE LOCATION>
 ```
 
 #### For platform specific archive
 
-1. From root project, run `./gradlew :release:archives:<platform>:<platform>Tar -Prelease`
-   * or from relevant platform project `gradle <platform>Tar -Prelease`
+1. From root project, run `./gradlew :release:archives:<platform>:<platform>Tar`
 2. Successful build generates archives in `release/archives/<platform>/build/distributions/`
 
 ##### Example for linux
@@ -35,16 +33,12 @@ For linux, the above steps will be
 // from root project
 ./gradlew :release:archives:linux:linuxTar -Prelease
 
-or
-// from linux project
-gradle linuxTar -Prelease
-
 cd release/archives/linux/build/distributions/
 
-tar -xzf opendistroforelasticsearch-data-prepper-jdk-<VERSION>-linux-<ARCHITECTURE>.tar.gz
-//e.g tar -xzf opendistroforelasticsearch-data-prepper-jdk-0.1.0-linux-x64.tar.gz
+tar -xzf opensearch-data-prepper-jdk-<VERSION>-linux-<ARCHITECTURE>.tar.gz
+//e.g tar -xzf opensearch-data-prepper-jdk-2.14.0-linux-x64.tar.gz
 
-./opendistroforelasticsearch-data-prepper-jdk-<VERSION>-linux-<ARCHITECTURE>/data-prepper-tar-install.sh <CONFIG FILE LOCATION>
+./opensearch-data-prepper-jdk-<VERSION>-linux-<ARCHITECTURE>/bin/data-prepper <CONFIG FILE LOCATION>
 
 ```
 
@@ -57,24 +51,24 @@ Also each platform specific project has a similar mechanism to upload platform s
 * From root project, run 
 
 ```
-./gradlew :release:archives:uploadArchives -Prelease -Pregion=us-east-1 -Pbucket=odfe-my-bucket -Pprofile=default
+./gradlew :release:archives:uploadArchives -Pregion=us-east-1 -Pbucket=data-prepper-my-bucket -Pprofile=default
 
 or
 
 //below command will use default values for region, bucket and profile
-./gradlew :release:archives:uploadArchives -Prelease 
+./gradlew :release:archives:uploadArchives 
 ```
 
 or
 
 * From current project, run
 
 ```
-./gradle uploadArchives -Prelease -Pregion=us-east-1 -Pbucket=odfe-my-bucket -Pprofile=default
+./gradle uploadArchives -Pregion=us-east-1 -Pbucket=data-prepper-my-bucket -Pprofile=default
 
 or
 
 //below command will use default values for region, bucket and profile
-./gradlew uploadArchives -Prelease 
+./gradlew uploadArchives 
 
 ```

release/archives/build.gradle

@@ -1,6 +1,10 @@
 /*
  * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
  */
 
 plugins {

release/archives/linux/build.gradle

@@ -1,6 +1,10 @@
 /*
  * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
  */
 
 //This file is intentionally left blank to support commands for this project

release/archives/linux/data-prepper-arm64.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+#
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+
+if [[ $# == 0 ]]; then
+    echo "Reading pipelines and data-prepper configuration files from Data Prepper home directory."
+elif [[ $# == 2 ]]; then
+    echo
+    echo "Data Prepper now supports reading pipeline and data-prepper configuration files"
+    echo "from Data Prepper home directory automatically."
+    echo "You can continue to specify paths to configuration files as command line arguments,"
+    echo "but that support will be dropped in a future release."
+    echo
+else
+    echo
+    echo "Error: Invalid number of arguments. Expected 0 or 2, received $#."
+    echo "Put configuration files in Data Prepper home directory and specify no arguments,"
+    echo "or specify paths to pipeline and data-prepper configuration files, for example:"
+    echo "bin/data-prepper config/example-pipelines.yaml config/example-data-prepper-config.yaml"
+    echo
+    exit 1
+fi
+
+MIN_REQ_JAVA_VERSION=11
+MIN_REQ_OPENJDK_VERSION=11
+DATA_PREPPER_BIN=$(dirname "$(readlink -f "$0")")
+DATA_PREPPER_HOME=`readlink -f "$DATA_PREPPER_BIN/.."`
+DATA_PREPPER_CLASSPATH="$DATA_PREPPER_HOME/lib/*"
+
+#check if java is installed
+if type -p java; then
+    _java=java
+elif [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]];  then
+    _java="$JAVA_HOME/bin/java"
+else
+    echo "java is required for executing data prepper, consider downloading data prepper tar with jdk"
+    exit 1
+fi
+
+if [[ "$_java" ]]
+then
+    java_type=$("$_java" -version 2>&1 | awk -F '"' '/version/ {print $1}')
+    java_version=$("$_java" -version 2>&1 | awk -F '"' '/version/ {print $2}' | sed 's/\([0-9]*\.[0-9]*\)\..*/\1/g')
+    echo "Found $java_type of $java_version"
+    if [[ $java_type == *"openjdk"* ]]
+    then
+        if (( $(echo "$java_version < $MIN_REQ_OPENJDK_VERSION" | bc -l) ))
+        then
+            echo "Minimum required for $java_type is $MIN_REQ_OPENJDK_VERSION"
+            exit 1
+        fi
+    else
+        if (( $(echo "$java_version < $MIN_REQ_JAVA_VERSION" | bc -l) ))
+        then
+            echo "Minimum required for $java_type is $MIN_REQ_JAVA_VERSION"
+            exit 1
+        fi
+    fi
+fi
+
+DATA_PREPPER_HOME_OPTS="-Ddata-prepper.dir=$DATA_PREPPER_HOME"
+DATA_PREPPER_JAVA_OPTS="-Dfile.encoding=UTF-8 -Dlog4j.configurationFile=$DATA_PREPPER_HOME/config/log4j2-rolling.properties"
+
+if [[ $# == 0 ]]; then
+    exec java $DATA_PREPPER_JAVA_OPTS $JAVA_OPTS $DATA_PREPPER_HOME_OPTS -cp "$DATA_PREPPER_CLASSPATH" org.opensearch.dataprepper.DataPrepperExecute
+else
+    PIPELINES_FILE_LOCATION=$1
+    CONFIG_FILE_LOCATION=$2
+    exec java $DATA_PREPPER_JAVA_OPTS $JAVA_OPTS $DATA_PREPPER_HOME_OPTS -cp "$DATA_PREPPER_CLASSPATH" org.opensearch.dataprepper.DataPrepperExecute $PIPELINES_FILE_LOCATION $CONFIG_FILE_LOCATION
+fi

release/archives/linux/data-prepper-jdk-arm64.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+#
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+
+if [[ $# == 0 ]]; then
+    echo "Reading pipelines and data-prepper configuration files from Data Prepper home directory."
+elif [[ $# == 2 ]]; then
+    echo
+    echo "Data Prepper now supports reading pipeline and data-prepper configuration files"
+    echo "from Data Prepper home directory automatically."
+    echo "You can continue to specify paths to configuration files as command line arguments,"
+    echo "but that support will be dropped in a future release."
+    echo
+else
+    echo
+    echo "Error: Invalid number of arguments. Expected 0 or 2, received $#."
+    echo "Put configuration files in Data Prepper home directory and specify no arguments,"
+    echo "or specify paths to pipeline and data-prepper configuration files, for example:"
+    echo "bin/data-prepper config/example-pipelines.yaml config/example-data-prepper-config.yaml"
+    echo
+    exit 1
+fi
+
+DATA_PREPPER_BIN=$(dirname "$(readlink -f "$0")")
+DATA_PREPPER_HOME=`readlink -f "$DATA_PREPPER_BIN/.."`
+DATA_PREPPER_CLASSPATH="$DATA_PREPPER_HOME/lib/*"
+OPENJDK=$(ls -1 $DATA_PREPPER_HOME/openjdk/ 2>/dev/null)
+
+export JAVA_HOME=$DATA_PREPPER_HOME/openjdk/$OPENJDK
+echo "JAVA_HOME is set to $JAVA_HOME"
+export PATH=$JAVA_HOME/bin:$PATH
+
+DATA_PREPPER_HOME_OPTS="-Ddata-prepper.dir=$DATA_PREPPER_HOME"
+DATA_PREPPER_JAVA_OPTS="-Dfile.encoding=UTF-8 -Dlog4j.configurationFile=$DATA_PREPPER_HOME/config/log4j2-rolling.properties"
+
+if [[ $# == 0 ]]; then
+    exec java $DATA_PREPPER_JAVA_OPTS $JAVA_OPTS $DATA_PREPPER_HOME_OPTS -cp "$DATA_PREPPER_CLASSPATH" org.opensearch.dataprepper.DataPrepperExecute
+else
+    PIPELINES_FILE_LOCATION=$1
+    CONFIG_FILE_LOCATION=$2
+    exec java $DATA_PREPPER_JAVA_OPTS $JAVA_OPTS $DATA_PREPPER_HOME_OPTS -cp "$DATA_PREPPER_CLASSPATH" org.opensearch.dataprepper.DataPrepperExecute $PIPELINES_FILE_LOCATION $CONFIG_FILE_LOCATION
+fi

release/archives/linux/data-prepper-jdk-x64.sh

@@ -4,6 +4,10 @@
 # Copyright OpenSearch Contributors
 # SPDX-License-Identifier: Apache-2.0
 #
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
 
 if [[ $# == 0 ]]; then
     echo "Reading pipelines and data-prepper configuration files from Data Prepper home directory."

release/archives/linux/data-prepper-x64.sh

@@ -4,6 +4,10 @@
 # Copyright OpenSearch Contributors
 # SPDX-License-Identifier: Apache-2.0
 #
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
 
 if [[ $# == 0 ]]; then
     echo "Reading pipelines and data-prepper configuration files from Data Prepper home directory."

release/build-resources.gradle

@@ -1,16 +1,20 @@
 /*
  * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
  */
 
 //preferably try to main the alphabetical order
 ext {
     architectures = [
-            linux: ['x64']
+            linux: ['x64', 'arm64']
     ]
     jdkSources = [
             linux_x64: 'https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.16%2B8/OpenJDK17U-jdk_x64_linux_hotspot_17.0.16_8.tar.gz',
-            linux_arm64: 'https://hg.openjdk.java.net/aarch64-port/jdk8/archive/tip.tar.gz'
+            linux_arm64: 'https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.17%2B10/OpenJDK17U-jdk_aarch64_linux_hotspot_17.0.17_10.tar.gz'
     ]
     awsResources = [
             default_region: 'us-east-1',

release/smoke-tests/README.md

@@ -35,11 +35,13 @@ To run automated smoke test on the default archive file you can use the followin
 ```
 
 You can also customize what it tests against. The `-i` parameter specifies a base Docker image. The `-t` parameter determines which tar archive file to use.
-The values for `-t` are `opensearch-data-prepper` or `opensearch-data-prepper-jdk`.
+The values for `-t` are `opensearch-data-prepper` or `opensearch-data-prepper-jdk`. The `-a` parameter specifies the architecture (`x64` or `arm64`, defaults to `x64`).
 
 ```shell
 ./release/smoke-tests/run-tarball-files-smoke-tests.sh -i eclipse-temurin:11 -t opensearch-data-prepper
 ./release/smoke-tests/run-tarball-files-smoke-tests.sh -i eclipse-temurin:17 -t opensearch-data-prepper
 ./release/smoke-tests/run-tarball-files-smoke-tests.sh -i ubuntu:latest -t opensearch-data-prepper-jdk
+./release/smoke-tests/run-tarball-files-smoke-tests.sh -a arm64 -t opensearch-data-prepper
+./release/smoke-tests/run-tarball-files-smoke-tests.sh -a arm64 -t opensearch-data-prepper-jdk -i ubuntu:latest
 ```
 

release/smoke-tests/data-prepper-tar/build.sh

@@ -1,7 +1,13 @@
 #!/bin/bash
 
+#
 # Copyright OpenSearch Contributors
 # SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
 
 set -e
 
@@ -31,6 +37,7 @@ if [ -z "${SMOKE_IMAGE_NAME}" ]; then
 fi
 
 docker build \
+    --platform="${PLATFORM:-linux/amd64}" \
     --build-arg BUILD_NAME="${BUILD_NAME}" \
     --build-arg DATA_PREPPER_VERSION="${DATA_PREPPER_VERSION}" \
     --build-arg DOCKER_FILE_DIR="${DOCKER_FILE_DIR}" \

release/smoke-tests/run-tarball-files-smoke-tests.sh

@@ -1,7 +1,13 @@
 #!/bin/bash
 
+#
 # Copyright OpenSearch Contributors
 # SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
 
 set -e
 
@@ -34,6 +40,11 @@ function usage() {
     echo -e "Required arguments:"
     echo -e "-v DATA_PREPPER_VERSION\tSpecify the Data Prepper build version to test such as '1.3.0-SNAPSHOT'."
     echo -e ""
+    echo -e "Optional arguments:"
+    echo -e "-a ARCHITECTURE\tSpecify the architecture to test (x64 or arm64). Default is x64."
+    echo -e "-i FROM_IMAGE\tSpecify the base Docker image to use. Default is eclipse-temurin:11."
+    echo -e "-t TAR_NAME\tSpecify the tar name prefix (opensearch-data-prepper or opensearch-data-prepper-jdk). Default is opensearch-data-prepper."
+    echo -e ""
     echo -e "Only one of the following argument sets can be used:"
     echo -e "Smoke test local file arguments:"
     echo -e "-d TAR_DIR\tSpecify local directory containing tarball files to be smoke tested."
@@ -48,6 +59,8 @@ function usage() {
     echo -e "--------------------------------------------------------------------------"
     echo -e "Examples, run from repo root directory"
     echo -e "\t\"./release/smoke-tests/run-tarball-files-smoke-tests.sh -v 1.3.0-SNAPSHOT -d release/archives/linux/build/distributions\""
+    echo -e "\t\"./release/smoke-tests/run-tarball-files-smoke-tests.sh -v 1.3.0-SNAPSHOT -a arm64 -d release/archives/linux/build/distributions\""
+    echo -e "\t\"./release/smoke-tests/run-tarball-files-smoke-tests.sh -v 1.3.0-SNAPSHOT -a arm64 -t opensearch-data-prepper-jdk -d release/archives/linux/build/distributions\""
     echo -e "\t\"./release/smoke-tests/run-tarball-files-smoke-tests.sh -v 1.3.0-SNAPSHOT -b staging-bucket -n 1\""
     echo -e "\t\"./release/smoke-tests/run-tarball-files-smoke-tests.sh -v 1.3.0-SNAPSHOT -u https://staging.opensearch.org -n 1\""
     echo -e ""
@@ -112,7 +125,7 @@ function get_url_tar() {
 }
 
 function run_smoke_test() {
-    export BUILD_NAME="${TAR_NAME}-${DATA_PREPPER_VERSION}-linux-x64"
+    export BUILD_NAME="${TAR_NAME}-${DATA_PREPPER_VERSION}-linux-${ARCHITECTURE}"
     export TAR_FILE="${BUILD_NAME}.tar.gz"
 
     case $TAR_SOURCE_TYPE in
@@ -142,8 +155,9 @@ function run_smoke_test() {
     cd "${CURRENT_DIR}" || exit
 }
 
-while getopts "b:d:hi:n:t:u:v:" arg; do
+while getopts "a:b:d:hi:n:t:u:v:" arg; do
     case $arg in
+        a) export ARCHITECTURE=$OPTARG;;
         b) export BUCKET_NAME=$OPTARG;;
         d) export TAR_DIR=$OPTARG;;
         h) usage;;
@@ -201,6 +215,18 @@ then
     export TAR_NAME="opensearch-data-prepper"
 fi
 
+if ! is_defined "${ARCHITECTURE}"
+then
+    export ARCHITECTURE="x64"
+fi
+
+# Set Docker platform based on architecture
+if [ "${ARCHITECTURE}" = "arm64" ]; then
+    export PLATFORM="linux/arm64"
+else
+    export PLATFORM="linux/amd64"
+fi
+
 
 echo -e "Using tarball source ${TAR_SOURCE_TYPE}"