fix: update pipeline and dev scripts (#578)

* fix scripts for bottlecap dev

* update `build_layer.sh`

it doesnt need incoming suffix because layers only use architecture and alpine will never be available

* rename `SUFFIX` to `FILE_SUFFIX`

* update compile scripts to calculate `FILE_SUFFIX`

* remove suffix for arch flavors

* update pipeline to remove `suffix` usage

* typo

* bring back suffix

we rely on alpine images :l

* bring back `FILE_SUFFIX` check

* remove appending `-alpine` programatically

* pass `FILE_SUFFIX` to build

* update path to the right one

Author: duncanista

.gitignore

@@ -26,4 +26,4 @@ local_tests/serverless-init/logs.txt
 bottlecap/target
 bottlecap/proptest-regressions
 
-.gitlab/pipeline-**
+.gitlab/pipeline*

.gitlab/scripts/build_layer.sh

@@ -12,9 +12,9 @@ if [ -z "$ARCHITECTURE" ]; then
     exit 1
 fi
 
-if [ -z "$SUFFIX" ]; then
-    printf "No suffix provided, using ${ARCHITECTURE}\n"
-    SUFFIX=$ARCHITECTURE
+if [ -z "$FILE_SUFFIX" ]; then
+    printf "[WARNING] No FILE_SUFFIX provided, using ${ARCHITECTURE}\n"
+    FILE_SUFFIX=$ARCHITECTURE
 fi
 
 # Move into the root directory, so this script can be called from any directory
@@ -24,7 +24,7 @@ cd $ROOT_DIR
 
 EXTENSION_DIR=".layers"
 TARGET_DIR=$(pwd)/$EXTENSION_DIR
-EXTENSION_PATH=$TARGET_DIR/datadog-extension-${SUFFIX}
+EXTENSION_PATH=$TARGET_DIR/datadog_extension-${FILE_SUFFIX}
 
 mkdir -p $EXTENSION_DIR
 rm -rf ${EXTENSION_PATH} 2>/dev/null
@@ -35,13 +35,13 @@ docker_build() {
     local arch=$1
 
     docker buildx build --platform linux/${arch} \
-        -t datadog/build-extension-${SUFFIX} \
+        -t datadog/build-extension-${FILE_SUFFIX} \
         -f ./images/Dockerfile.build_layer \
-        --build-arg SUFFIX=$SUFFIX \
+        --build-arg FILE_SUFFIX=$FILE_SUFFIX \
         . -o $EXTENSION_PATH
 
-    cp $EXTENSION_PATH/datadog_extension.zip $TARGET_DIR/datadog_extension-${SUFFIX}.zip
-    unzip $EXTENSION_PATH/datadog_extension.zip -d $TARGET_DIR/datadog_extension-${SUFFIX}
+    cp $EXTENSION_PATH/datadog_extension.zip $TARGET_DIR/datadog_extension-${FILE_SUFFIX}.zip
+    unzip $EXTENSION_PATH/datadog_extension.zip -d $TARGET_DIR/datadog_extension-${FILE_SUFFIX}
     rm -rf ${EXTENSION_PATH}
 }
 

.gitlab/scripts/compile_bottlecap.sh

@@ -12,18 +12,18 @@ if [ -z "$ARCHITECTURE" ]; then
     exit 1
 fi
 
+if [ -z "$FILE_SUFFIX" ]; then
+    printf "[WARNING] No FILE_SUFFIX provided, using ${ARCHITECTURE}\n"
+    FILE_SUFFIX=$ARCHITECTURE
+fi
+
 if [ -z "$ALPINE" ]; then
     printf "[ERROR]: ALPINE not specified\n"
     exit 1
 else
     printf "Alpine compile requested: ${ALPINE}\n"
 fi
 
-if [ -z "$SUFFIX" ]; then
-    printf "No suffix provided, using ${ARCHITECTURE}\n"
-    SUFFIX=$ARCHITECTURE
-fi
-
 if [ "$ALPINE" = "0" ]; then
     COMPILE_IMAGE=Dockerfile.bottlecap.compile
 else
@@ -39,7 +39,7 @@ cd $ROOT_DIR
 
 BINARY_DIR=".binaries"
 TARGET_DIR=$(pwd)/$BINARY_DIR
-BINARY_PATH=$TARGET_DIR/compiled-bottlecap-${SUFFIX}
+BINARY_PATH=$TARGET_DIR/compiled-bottlecap-$FILE_SUFFIX
 
 mkdir -p $BINARY_DIR
 
@@ -55,14 +55,14 @@ docker_build() {
     fi
 
     docker buildx build --platform linux/${arch} \
-        -t datadog/compile-bottlecap-${SUFFIX} \
+        -t datadog/compile-bottlecap \
         -f ./images/${file} \
         --build-arg PLATFORM=$PLATFORM \
         . -o $BINARY_PATH
 
     # Copy the compiled binary to the target directory with the expected name
     # If it already exist, it will be replaced
-    cp $BINARY_PATH/bottlecap $TARGET_DIR/bottlecap-${SUFFIX}
+    cp $BINARY_PATH/bottlecap $TARGET_DIR/bottlecap-$FILE_SUFFIX
 }
 
 docker_build $ARCHITECTURE $COMPILE_IMAGE

.gitlab/scripts/compile_go_agent.sh

@@ -12,6 +12,11 @@ if [ -z "$ARCHITECTURE" ]; then
     exit 1
 fi
 
+if [ -z "$FILE_SUFFIX" ]; then
+    printf "[WARNING] No FILE_SUFFIX provided, using ${ARCHITECTURE}\n"
+    FILE_SUFFIX=$ARCHITECTURE
+fi
+
 if [ -z "$AGENT_VERSION" ]; then
     printf "[ERROR]: AGENT_VERSION not specified\n"
     exit 1
@@ -44,11 +49,6 @@ else
     COMPILE_IMAGE=Dockerfile.go_agent.alpine.compile
 fi
 
-if [ -z "$SUFFIX" ]; then
-    printf "No suffix provided, using ${ARCHITECTURE}\n"
-    SUFFIX=$ARCHITECTURE
-fi
-
 # Allow override build tags
 if [ -z "$BUILD_TAGS" ]; then
     BUILD_TAGS="serverless otlp"
@@ -63,7 +63,7 @@ MAIN_DIR=$(pwd) # datadog-lambda-extension
 
 BINARY_DIR=".binaries"
 TARGET_DIR=$MAIN_DIR/$BINARY_DIR
-BINARY_PATH=$TARGET_DIR/compiled-datadog-agent-${SUFFIX}
+BINARY_PATH=$TARGET_DIR/compiled-datadog-agent-$FILE_SUFFIX
 
 mkdir -p $BINARY_DIR
 
@@ -84,7 +84,7 @@ function docker_compile {
     file=$2
 
     docker buildx build --platform linux/${arch} \
-        -t datadog/compile-go-agent-${SUFFIX}:${VERSION} \
+        -t datadog/compile-go-agent:${VERSION} \
         -f ${MAIN_DIR}/images/${file} \
         --build-arg EXTENSION_VERSION="${VERSION}" \
         --build-arg AGENT_VERSION="${AGENT_VERSION}" \
@@ -93,7 +93,7 @@ function docker_compile {
 
     # Copy the compiled binary to the target directory with the expected name
     # If it already exist, it will be replaced
-    cp $BINARY_PATH/datadog-agent $TARGET_DIR/datadog-agent-${SUFFIX}
+    cp $BINARY_PATH/datadog-agent $TARGET_DIR/datadog-agent-$FILE_SUFFIX
 }
 
 docker_compile $ARCHITECTURE $COMPILE_IMAGE

.gitlab/templates/pipeline.yaml.tpl

@@ -58,7 +58,7 @@ go agent ({{ $flavor.name }}):
   variables:
     ARCHITECTURE: {{ $flavor.arch }}
     ALPINE: {{ $flavor.alpine }}
-    SUFFIX: {{ $flavor.suffix }}
+    FILE_SUFFIX: {{ $flavor.suffix }}
   script:
     - echo "Building go agent based on $AGENT_BRANCH"
     # TODO: do this clone once in a separate job so that we can make sure that
@@ -79,7 +79,7 @@ bottlecap ({{ $flavor.name }}):
   variables:
     ARCHITECTURE: {{ $flavor.arch }}
     ALPINE: {{ $flavor.alpine }}
-    SUFFIX: {{ $flavor.suffix }}
+    FILE_SUFFIX: {{ $flavor.suffix }}
   script:
     - .gitlab/scripts/compile_bottlecap.sh
 
@@ -103,7 +103,7 @@ layer ({{ $flavor.name }}):
       - .layers/datadog_extension-{{ $flavor.suffix }}/*
   variables:
     ARCHITECTURE: {{ $flavor.arch }}
-    SUFFIX: {{ $flavor.suffix }}
+    FILE_SUFFIX: {{ $flavor.suffix }}
   script:
     - .gitlab/scripts/build_layer.sh
 

images/Dockerfile.bottlecap.dev

@@ -1,5 +1,5 @@
 FROM ubuntu:22.04 AS compresser
-ARG SUFFIX
+ARG FILE_SUFFIX
 
 RUN apt-get update && apt-get install -y zip binutils
 
@@ -8,7 +8,7 @@ COPY --from=public.ecr.aws/datadog/lambda-extension:latest /opt/datadog-agent-go
 RUN mkdir /extensions
 WORKDIR /extensions
 
-COPY .binaries/bottlecap-${SUFFIX} /extensions/datadog-agent
+COPY .binaries/bottlecap-${FILE_SUFFIX} /extensions/datadog-agent
 COPY ./scripts/datadog_wrapper /datadog_wrapper
 RUN chmod +x /datadog_wrapper
 RUN  zip -r datadog_extension.zip /extensions /datadog_wrapper /datadog-agent-go

images/Dockerfile.build_layer

@@ -1,13 +1,13 @@
 FROM ubuntu:22.04 AS compresser
 ARG DATADOG_WRAPPER=datadog_wrapper
-ARG SUFFIX
+ARG FILE_SUFFIX
 
 # Install dependencies
 RUN apt-get update
 RUN apt-get install -y zip binutils
 
 # Copy Go Agent binary
-COPY .binaries/datadog-agent-$SUFFIX /datadog-agent-go
+COPY .binaries/datadog-agent-$FILE_SUFFIX /datadog-agent-go
 
 # UPX compress on x86_64
 RUN if [ "$PLATFORM" = "x86_64" ]; then apt-get install -y upx=3.96-r0 && upx -1 /datadog-agent-go; fi
@@ -16,7 +16,7 @@ RUN mkdir /extensions
 WORKDIR /extensions
 
 # Copy Rust Agent binary
-COPY .binaries/bottlecap-$SUFFIX /extensions/datadog-agent
+COPY .binaries/bottlecap-$FILE_SUFFIX /extensions/datadog-agent
 
 # Copy wrapper script
 COPY ./scripts/$DATADOG_WRAPPER /$DATADOG_WRAPPER

scripts/build_bottlecap_layer.sh

@@ -17,10 +17,7 @@ if [ -z "$ARCHITECTURE" ]; then
     exit 1
 fi
 
-if [ -z "$SUFFIX" ]; then
-    printf "No suffix provided, using ${ARCHITECTURE}\n"
-    SUFFIX=$ARCHITECTURE
-fi
+FILE_SUFFIX=$ARCHITECTURE
 
 PLATFORM="aarch64"
 if [ "$ARCHITECTURE" == "amd64" ]; then
@@ -31,19 +28,25 @@ SCRIPTS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
 ROOT_DIR=$SCRIPTS_DIR/..
 cd $ROOT_DIR
 
-EXTENSION_DIR=".layers"
-EXTENSION_PATH=$ROOT_DIR/$EXTENSION_DIR
+LAYERS_DIR=".layers"
+BUILD_DIR=$ROOT_DIR/$LAYERS_DIR
+EXTENSION_PATH=$BUILD_DIR/datadog_extension-$FILE_SUFFIX
+
+mkdir -p $LAYERS_DIR
+rm -rf ${EXTENSION_PATH} 2>/dev/null
+
+cd $ROOT_DIR
 
 ALPINE=0
-ARCHITECTURE=$ARCHITECTURE ALPINE=$ALPINE .gitlab/scripts/compile_bottlecap.sh
+ARCHITECTURE=$ARCHITECTURE ALPINE=$ALPINE FILE_SUFFIX=$FILE_SUFFIX .gitlab/scripts/compile_bottlecap.sh
 
 docker buildx build --platform linux/${ARCHITECTURE} \
-    -t datadog/build-bottlecap-${SUFFIX} \
+    -t datadog/build-bottlecap-${FILE_SUFFIX} \
     -f ./images/Dockerfile.bottlecap.dev \
-    --build-arg SUFFIX=$SUFFIX \
-    . -o $EXTENSION_PATH/datadog_bottlecap-${SUFFIX}
+    --build-arg FILE_SUFFIX=$FILE_SUFFIX \
+    . -o $EXTENSION_PATH
 
-cp $EXTENSION_PATH/datadog_bottlecap-${SUFFIX}/datadog_extension.zip $EXTENSION_PATH/datadog_bottlecap-${SUFFIX}.zip
+cp $EXTENSION_PATH/datadog_extension.zip $EXTENSION_PATH.zip
 
-unzip $EXTENSION_PATH/datadog_bottlecap-${SUFFIX}/datadog_extension.zip -d $EXTENSION_PATH/datadog_bottlecap-${SUFFIX}
-rm -rf $EXTENSION_PATH/datadog_bottlecap-${SUFFIX}/datadog_extension.zip
+unzip $EXTENSION_PATH/datadog_extension.zip -d $EXTENSION_PATH
+rm -rf $EXTENSION_PATH/datadog_extension.zip

scripts/publish_bottlecap_sandbox.sh

@@ -17,11 +17,11 @@ set -e
 _init_arg(){
   if [ "$ARCHITECTURE" == "amd64" ]; then
       echo "Publishing for amd64 only"
-      LAYER_PATH=".layers/datadog_bottlecap-amd64.zip"
+      LAYER_PATH=".layers/datadog_extension-amd64.zip"
       LAYER_NAME="Datadog-Bottlecap-Beta"
   elif [ "$ARCHITECTURE" == "arm64" ]; then
       echo "Publishing for arm64 only"
-      LAYER_PATH=".layers/datadog_bottlecap-arm64.zip"
+      LAYER_PATH=".layers/datadog_extension-arm64.zip"
       LAYER_NAME="Datadog-Bottlecap-Beta-ARM"
   fi
   if [ -z $ARCHITECTURE ]; then
@@ -40,10 +40,6 @@ _init_arg(){
 }
 
 publish(){
-  # Move into the root directory
-  SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-  cd $SCRIPTS_DIR/..
-
   ARCHITECTURE=$ARCHITECTURE ./scripts/build_bottlecap_layer.sh
   NEW_VERSION=$(aws-vault exec sso-serverless-sandbox-account-admin -- aws lambda publish-layer-version --layer-name "${LAYER_NAME}" \
       --description "Datadog Bottlecap Beta" \