diff --git a/release/archives/README.md b/release/archives/README.md index c1e03f1b41..48206479af 100644 --- a/release/archives/README.md +++ b/release/archives/README.md @@ -3,6 +3,7 @@ **Supported Platform** * Linux - x64 +* Linux - arm64 ### General Instructions @@ -11,21 +12,18 @@ To build all archives, check out the corresponding branch for the version and follow below steps 1. Building an archive depends on the platform on which it will be executed/run -2. For building archives for all supported platforms, - * from root project run `./gradlew :release:archives:buildArchives -Prelease` - * or from current project `gradle buildArchives -Prelease` +2. For building archives for all supported platforms, from root project run `./gradlew :release:archives:buildArchives` 3. Successful build will generate archives in `release/archives//build/distributions/` directory 4. Generated archives includes a script file which can be used to execute the data prepper using ``` -tar -xzf opendistroforelasticsearch-data-prepper-jdk-v--.tar.gz -./opendistroforelasticsearch-data-prepper-jdk-v--/data-prepper-tar-install.sh +tar -xzf opensearch-data-prepper-jdk-v--.tar.gz +./opensearch-data-prepper-jdk-v--/bin/data-prepper ``` #### For platform specific archive -1. From root project, run `./gradlew :release:archives::Tar -Prelease` - * or from relevant platform project `gradle Tar -Prelease` +1. From root project, run `./gradlew :release:archives::Tar` 2. Successful build generates archives in `release/archives//build/distributions/` ##### Example for linux @@ -35,16 +33,12 @@ For linux, the above steps will be // from root project ./gradlew :release:archives:linux:linuxTar -Prelease -or -// from linux project -gradle linuxTar -Prelease - cd release/archives/linux/build/distributions/ -tar -xzf opendistroforelasticsearch-data-prepper-jdk--linux-.tar.gz -//e.g tar -xzf opendistroforelasticsearch-data-prepper-jdk-0.1.0-linux-x64.tar.gz +tar -xzf opensearch-data-prepper-jdk--linux-.tar.gz +//e.g tar -xzf opensearch-data-prepper-jdk-2.14.0-linux-x64.tar.gz -./opendistroforelasticsearch-data-prepper-jdk--linux-/data-prepper-tar-install.sh +./opensearch-data-prepper-jdk--linux-/bin/data-prepper ``` @@ -57,12 +51,12 @@ Also each platform specific project has a similar mechanism to upload platform s * From root project, run ``` -./gradlew :release:archives:uploadArchives -Prelease -Pregion=us-east-1 -Pbucket=odfe-my-bucket -Pprofile=default +./gradlew :release:archives:uploadArchives -Pregion=us-east-1 -Pbucket=data-prepper-my-bucket -Pprofile=default or //below command will use default values for region, bucket and profile -./gradlew :release:archives:uploadArchives -Prelease +./gradlew :release:archives:uploadArchives ``` or @@ -70,11 +64,11 @@ or * From current project, run ``` -./gradle uploadArchives -Prelease -Pregion=us-east-1 -Pbucket=odfe-my-bucket -Pprofile=default +./gradle uploadArchives -Pregion=us-east-1 -Pbucket=data-prepper-my-bucket -Pprofile=default or //below command will use default values for region, bucket and profile -./gradlew uploadArchives -Prelease +./gradlew uploadArchives ``` diff --git a/release/archives/build.gradle b/release/archives/build.gradle index bb5ec3f53a..d9c3472944 100644 --- a/release/archives/build.gradle +++ b/release/archives/build.gradle @@ -1,6 +1,10 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ plugins { diff --git a/release/archives/linux/build.gradle b/release/archives/linux/build.gradle index 24b3479d4e..df27f42a4d 100644 --- a/release/archives/linux/build.gradle +++ b/release/archives/linux/build.gradle @@ -1,6 +1,10 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ //This file is intentionally left blank to support commands for this project \ No newline at end of file diff --git a/release/archives/linux/data-prepper-arm64.sh b/release/archives/linux/data-prepper-arm64.sh new file mode 100755 index 0000000000..e847880796 --- /dev/null +++ b/release/archives/linux/data-prepper-arm64.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +if [[ $# == 0 ]]; then + echo "Reading pipelines and data-prepper configuration files from Data Prepper home directory." +elif [[ $# == 2 ]]; then + echo + echo "Data Prepper now supports reading pipeline and data-prepper configuration files" + echo "from Data Prepper home directory automatically." + echo "You can continue to specify paths to configuration files as command line arguments," + echo "but that support will be dropped in a future release." + echo +else + echo + echo "Error: Invalid number of arguments. Expected 0 or 2, received $#." + echo "Put configuration files in Data Prepper home directory and specify no arguments," + echo "or specify paths to pipeline and data-prepper configuration files, for example:" + echo "bin/data-prepper config/example-pipelines.yaml config/example-data-prepper-config.yaml" + echo + exit 1 +fi + +MIN_REQ_JAVA_VERSION=11 +MIN_REQ_OPENJDK_VERSION=11 +DATA_PREPPER_BIN=$(dirname "$(readlink -f "$0")") +DATA_PREPPER_HOME=`readlink -f "$DATA_PREPPER_BIN/.."` +DATA_PREPPER_CLASSPATH="$DATA_PREPPER_HOME/lib/*" + +#check if java is installed +if type -p java; then + _java=java +elif [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]]; then + _java="$JAVA_HOME/bin/java" +else + echo "java is required for executing data prepper, consider downloading data prepper tar with jdk" + exit 1 +fi + +if [[ "$_java" ]] +then + java_type=$("$_java" -version 2>&1 | awk -F '"' '/version/ {print $1}') + java_version=$("$_java" -version 2>&1 | awk -F '"' '/version/ {print $2}' | sed 's/\([0-9]*\.[0-9]*\)\..*/\1/g') + echo "Found $java_type of $java_version" + if [[ $java_type == *"openjdk"* ]] + then + if (( $(echo "$java_version < $MIN_REQ_OPENJDK_VERSION" | bc -l) )) + then + echo "Minimum required for $java_type is $MIN_REQ_OPENJDK_VERSION" + exit 1 + fi + else + if (( $(echo "$java_version < $MIN_REQ_JAVA_VERSION" | bc -l) )) + then + echo "Minimum required for $java_type is $MIN_REQ_JAVA_VERSION" + exit 1 + fi + fi +fi + +DATA_PREPPER_HOME_OPTS="-Ddata-prepper.dir=$DATA_PREPPER_HOME" +DATA_PREPPER_JAVA_OPTS="-Dfile.encoding=UTF-8 -Dlog4j.configurationFile=$DATA_PREPPER_HOME/config/log4j2-rolling.properties" + +if [[ $# == 0 ]]; then + exec java $DATA_PREPPER_JAVA_OPTS $JAVA_OPTS $DATA_PREPPER_HOME_OPTS -cp "$DATA_PREPPER_CLASSPATH" org.opensearch.dataprepper.DataPrepperExecute +else + PIPELINES_FILE_LOCATION=$1 + CONFIG_FILE_LOCATION=$2 + exec java $DATA_PREPPER_JAVA_OPTS $JAVA_OPTS $DATA_PREPPER_HOME_OPTS -cp "$DATA_PREPPER_CLASSPATH" org.opensearch.dataprepper.DataPrepperExecute $PIPELINES_FILE_LOCATION $CONFIG_FILE_LOCATION +fi diff --git a/release/archives/linux/data-prepper-jdk-arm64.sh b/release/archives/linux/data-prepper-jdk-arm64.sh new file mode 100755 index 0000000000..5eff4ffad0 --- /dev/null +++ b/release/archives/linux/data-prepper-jdk-arm64.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +if [[ $# == 0 ]]; then + echo "Reading pipelines and data-prepper configuration files from Data Prepper home directory." +elif [[ $# == 2 ]]; then + echo + echo "Data Prepper now supports reading pipeline and data-prepper configuration files" + echo "from Data Prepper home directory automatically." + echo "You can continue to specify paths to configuration files as command line arguments," + echo "but that support will be dropped in a future release." + echo +else + echo + echo "Error: Invalid number of arguments. Expected 0 or 2, received $#." + echo "Put configuration files in Data Prepper home directory and specify no arguments," + echo "or specify paths to pipeline and data-prepper configuration files, for example:" + echo "bin/data-prepper config/example-pipelines.yaml config/example-data-prepper-config.yaml" + echo + exit 1 +fi + +DATA_PREPPER_BIN=$(dirname "$(readlink -f "$0")") +DATA_PREPPER_HOME=`readlink -f "$DATA_PREPPER_BIN/.."` +DATA_PREPPER_CLASSPATH="$DATA_PREPPER_HOME/lib/*" +OPENJDK=$(ls -1 $DATA_PREPPER_HOME/openjdk/ 2>/dev/null) + +export JAVA_HOME=$DATA_PREPPER_HOME/openjdk/$OPENJDK +echo "JAVA_HOME is set to $JAVA_HOME" +export PATH=$JAVA_HOME/bin:$PATH + +DATA_PREPPER_HOME_OPTS="-Ddata-prepper.dir=$DATA_PREPPER_HOME" +DATA_PREPPER_JAVA_OPTS="-Dfile.encoding=UTF-8 -Dlog4j.configurationFile=$DATA_PREPPER_HOME/config/log4j2-rolling.properties" + +if [[ $# == 0 ]]; then + exec java $DATA_PREPPER_JAVA_OPTS $JAVA_OPTS $DATA_PREPPER_HOME_OPTS -cp "$DATA_PREPPER_CLASSPATH" org.opensearch.dataprepper.DataPrepperExecute +else + PIPELINES_FILE_LOCATION=$1 + CONFIG_FILE_LOCATION=$2 + exec java $DATA_PREPPER_JAVA_OPTS $JAVA_OPTS $DATA_PREPPER_HOME_OPTS -cp "$DATA_PREPPER_CLASSPATH" org.opensearch.dataprepper.DataPrepperExecute $PIPELINES_FILE_LOCATION $CONFIG_FILE_LOCATION +fi diff --git a/release/archives/linux/data-prepper-jdk-x64.sh b/release/archives/linux/data-prepper-jdk-x64.sh index 18c8951b09..5eff4ffad0 100644 --- a/release/archives/linux/data-prepper-jdk-x64.sh +++ b/release/archives/linux/data-prepper-jdk-x64.sh @@ -4,6 +4,10 @@ # Copyright OpenSearch Contributors # SPDX-License-Identifier: Apache-2.0 # +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# if [[ $# == 0 ]]; then echo "Reading pipelines and data-prepper configuration files from Data Prepper home directory." diff --git a/release/archives/linux/data-prepper-x64.sh b/release/archives/linux/data-prepper-x64.sh index 6650702670..e847880796 100755 --- a/release/archives/linux/data-prepper-x64.sh +++ b/release/archives/linux/data-prepper-x64.sh @@ -4,6 +4,10 @@ # Copyright OpenSearch Contributors # SPDX-License-Identifier: Apache-2.0 # +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# if [[ $# == 0 ]]; then echo "Reading pipelines and data-prepper configuration files from Data Prepper home directory." diff --git a/release/build-resources.gradle b/release/build-resources.gradle index 5144de25d8..11d7680ed0 100644 --- a/release/build-resources.gradle +++ b/release/build-resources.gradle @@ -1,16 +1,20 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ //preferably try to main the alphabetical order ext { architectures = [ - linux: ['x64'] + linux: ['x64', 'arm64'] ] jdkSources = [ linux_x64: 'https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.16%2B8/OpenJDK17U-jdk_x64_linux_hotspot_17.0.16_8.tar.gz', - linux_arm64: 'https://hg.openjdk.java.net/aarch64-port/jdk8/archive/tip.tar.gz' + linux_arm64: 'https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.17%2B10/OpenJDK17U-jdk_aarch64_linux_hotspot_17.0.17_10.tar.gz' ] awsResources = [ default_region: 'us-east-1', diff --git a/release/smoke-tests/README.md b/release/smoke-tests/README.md index 43e523d262..1e6a293060 100644 --- a/release/smoke-tests/README.md +++ b/release/smoke-tests/README.md @@ -35,11 +35,13 @@ To run automated smoke test on the default archive file you can use the followin ``` You can also customize what it tests against. The `-i` parameter specifies a base Docker image. The `-t` parameter determines which tar archive file to use. -The values for `-t` are `opensearch-data-prepper` or `opensearch-data-prepper-jdk`. +The values for `-t` are `opensearch-data-prepper` or `opensearch-data-prepper-jdk`. The `-a` parameter specifies the architecture (`x64` or `arm64`, defaults to `x64`). ```shell ./release/smoke-tests/run-tarball-files-smoke-tests.sh -i eclipse-temurin:11 -t opensearch-data-prepper ./release/smoke-tests/run-tarball-files-smoke-tests.sh -i eclipse-temurin:17 -t opensearch-data-prepper ./release/smoke-tests/run-tarball-files-smoke-tests.sh -i ubuntu:latest -t opensearch-data-prepper-jdk +./release/smoke-tests/run-tarball-files-smoke-tests.sh -a arm64 -t opensearch-data-prepper +./release/smoke-tests/run-tarball-files-smoke-tests.sh -a arm64 -t opensearch-data-prepper-jdk -i ubuntu:latest ``` diff --git a/release/smoke-tests/data-prepper-tar/build.sh b/release/smoke-tests/data-prepper-tar/build.sh index 3e1a4218cf..bb876046c1 100755 --- a/release/smoke-tests/data-prepper-tar/build.sh +++ b/release/smoke-tests/data-prepper-tar/build.sh @@ -1,7 +1,13 @@ #!/bin/bash +# # Copyright OpenSearch Contributors # SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# set -e @@ -31,6 +37,7 @@ if [ -z "${SMOKE_IMAGE_NAME}" ]; then fi docker build \ + --platform="${PLATFORM:-linux/amd64}" \ --build-arg BUILD_NAME="${BUILD_NAME}" \ --build-arg DATA_PREPPER_VERSION="${DATA_PREPPER_VERSION}" \ --build-arg DOCKER_FILE_DIR="${DOCKER_FILE_DIR}" \ diff --git a/release/smoke-tests/run-tarball-files-smoke-tests.sh b/release/smoke-tests/run-tarball-files-smoke-tests.sh index c7ab7db822..50f03d8c0a 100755 --- a/release/smoke-tests/run-tarball-files-smoke-tests.sh +++ b/release/smoke-tests/run-tarball-files-smoke-tests.sh @@ -1,7 +1,13 @@ #!/bin/bash +# # Copyright OpenSearch Contributors # SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# set -e @@ -34,6 +40,11 @@ function usage() { echo -e "Required arguments:" echo -e "-v DATA_PREPPER_VERSION\tSpecify the Data Prepper build version to test such as '1.3.0-SNAPSHOT'." echo -e "" + echo -e "Optional arguments:" + echo -e "-a ARCHITECTURE\tSpecify the architecture to test (x64 or arm64). Default is x64." + echo -e "-i FROM_IMAGE\tSpecify the base Docker image to use. Default is eclipse-temurin:11." + echo -e "-t TAR_NAME\tSpecify the tar name prefix (opensearch-data-prepper or opensearch-data-prepper-jdk). Default is opensearch-data-prepper." + echo -e "" echo -e "Only one of the following argument sets can be used:" echo -e "Smoke test local file arguments:" echo -e "-d TAR_DIR\tSpecify local directory containing tarball files to be smoke tested." @@ -48,6 +59,8 @@ function usage() { echo -e "--------------------------------------------------------------------------" echo -e "Examples, run from repo root directory" echo -e "\t\"./release/smoke-tests/run-tarball-files-smoke-tests.sh -v 1.3.0-SNAPSHOT -d release/archives/linux/build/distributions\"" + echo -e "\t\"./release/smoke-tests/run-tarball-files-smoke-tests.sh -v 1.3.0-SNAPSHOT -a arm64 -d release/archives/linux/build/distributions\"" + echo -e "\t\"./release/smoke-tests/run-tarball-files-smoke-tests.sh -v 1.3.0-SNAPSHOT -a arm64 -t opensearch-data-prepper-jdk -d release/archives/linux/build/distributions\"" echo -e "\t\"./release/smoke-tests/run-tarball-files-smoke-tests.sh -v 1.3.0-SNAPSHOT -b staging-bucket -n 1\"" echo -e "\t\"./release/smoke-tests/run-tarball-files-smoke-tests.sh -v 1.3.0-SNAPSHOT -u https://staging.opensearch.org -n 1\"" echo -e "" @@ -112,7 +125,7 @@ function get_url_tar() { } function run_smoke_test() { - export BUILD_NAME="${TAR_NAME}-${DATA_PREPPER_VERSION}-linux-x64" + export BUILD_NAME="${TAR_NAME}-${DATA_PREPPER_VERSION}-linux-${ARCHITECTURE}" export TAR_FILE="${BUILD_NAME}.tar.gz" case $TAR_SOURCE_TYPE in @@ -142,8 +155,9 @@ function run_smoke_test() { cd "${CURRENT_DIR}" || exit } -while getopts "b:d:hi:n:t:u:v:" arg; do +while getopts "a:b:d:hi:n:t:u:v:" arg; do case $arg in + a) export ARCHITECTURE=$OPTARG;; b) export BUCKET_NAME=$OPTARG;; d) export TAR_DIR=$OPTARG;; h) usage;; @@ -201,6 +215,18 @@ then export TAR_NAME="opensearch-data-prepper" fi +if ! is_defined "${ARCHITECTURE}" +then + export ARCHITECTURE="x64" +fi + +# Set Docker platform based on architecture +if [ "${ARCHITECTURE}" = "arm64" ]; then + export PLATFORM="linux/arm64" +else + export PLATFORM="linux/amd64" +fi + echo -e "Using tarball source ${TAR_SOURCE_TYPE}"