chore: consolidate scripts (#568)

* remove unused files and directories

* update images to have explanation on docker layer steps

* update `feature_request_template.md`

* move `Dockerfile`s to `images` folder

* remove `build_and_publish_docker_image.sh`

* remove more unused scripts

* add check for `SUFFIX` on `compile_*` scripts

* update and move bottlecap dev image

* update `compile` scripts

* make `build_layer.sh` to use suffix check

* simplify bottlecap dev script

* removed unused script and image

Author: duncanista

.github/feature_request_template.md

@@ -17,8 +17,8 @@ about: Suggest an idea
 
 ## Specifications
 
-  - Datadog Lambda Layer version: 
-  - Python version: 
+  - Datadog Extension version:
+  - Datadog Lambda Library name and version:
 
 ## Stacktrace
 

.gitignore

@@ -1,5 +1,6 @@
 .DS_Store
 .layers
+.binaries
 .idea
 .serverless
 response.x

.gitlab/scripts/build_image.sh

@@ -40,7 +40,7 @@ fi
 
 docker buildx build \
     --platform $PLATFORM \
-    -f .gitlab/scripts/${TARGET_IMAGE} \
+    -f ./images/${TARGET_IMAGE} \
     --tag "$DOCKER_TARGET_IMAGE:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}${SUFFIX}" \
     --push .
 printf "Image built and pushed to $DOCKER_TARGET_IMAGE:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}${SUFFIX} for $PLATFORM\n"

.gitlab/scripts/build_layer.sh

@@ -12,37 +12,37 @@ if [ -z "$ARCHITECTURE" ]; then
     exit 1
 fi
 
-prepare_folders() {
-    # Move into the root directory, so this script can be called from any directory
-    SCRIPTS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
-    ROOT_DIR=$SCRIPTS_DIR/../..
-    cd $ROOT_DIR
-
-    echo $ROOT_DIR
+if [ -z "$SUFFIX" ]; then
+    printf "No suffix provided, using ${ARCHITECTURE}\n"
+    SUFFIX=$ARCHITECTURE
+fi
 
-    EXTENSION_DIR=".layers"
-    TARGET_DIR=$(pwd)/$EXTENSION_DIR
+# Move into the root directory, so this script can be called from any directory
+SCRIPTS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+ROOT_DIR=$SCRIPTS_DIR/../..
+cd $ROOT_DIR
 
-    rm -rf ${EXTENSION_DIR} 2>/dev/null
-    mkdir -p $EXTENSION_DIR
+EXTENSION_DIR=".layers"
+TARGET_DIR=$(pwd)/$EXTENSION_DIR
+EXTENSION_PATH=$TARGET_DIR/datadog-extension-${SUFFIX}
 
-    cd $ROOT_DIR
-}
+mkdir -p $EXTENSION_DIR
+rm -rf ${EXTENSION_PATH} 2>/dev/null
 
+cd $ROOT_DIR
 
 docker_build() {
     local arch=$1
 
     docker buildx build --platform linux/${arch} \
         -t datadog/build-extension-${SUFFIX} \
-        -f .gitlab/scripts/Dockerfile.build_layer \
+        -f ./images/Dockerfile.build_layer \
         --build-arg SUFFIX=$SUFFIX \
-        . -o $TARGET_DIR/datadog-extension-${SUFFIX}
+        . -o $EXTENSION_PATH
 
-    cp $TARGET_DIR/datadog-extension-${SUFFIX}/datadog_extension.zip $TARGET_DIR/datadog_extension-${SUFFIX}.zip
-    unzip $TARGET_DIR/datadog-extension-${SUFFIX}/datadog_extension.zip -d $TARGET_DIR/datadog_extension-${SUFFIX}
-    rm -rf $TARGET_DIR/datadog-extension-${SUFFIX}/
+    cp $EXTENSION_PATH/datadog_extension.zip $TARGET_DIR/datadog_extension-${SUFFIX}.zip
+    unzip $EXTENSION_PATH/datadog_extension.zip -d $TARGET_DIR/datadog_extension-${SUFFIX}
+    rm -rf ${EXTENSION_PATH}
 }
 
-prepare_folders
 docker_build $ARCHITECTURE

.gitlab/scripts/build_private_image.sh

@@ -44,7 +44,7 @@ printf "Tagging container image with version: $VERSION and latest\n"
 
 docker buildx build \
     --platform $PLATFORM \
-    -f .gitlab/scripts/${TARGET_IMAGE} \
+    -f ./images/${TARGET_IMAGE} \
     --tag "$DOCKER_TARGET_IMAGE:${IMAGE_TAG}${SUFFIX}" \
     --tag "$DOCKER_TARGET_IMAGE:${VERSION}${SUFFIX}" \
     --push .

.gitlab/scripts/compile_bottlecap.sh

@@ -19,31 +19,31 @@ else
     printf "Alpine compile requested: ${ALPINE}\n"
 fi
 
+if [ -z "$SUFFIX" ]; then
+    printf "No suffix provided, using ${ARCHITECTURE}\n"
+    SUFFIX=$ARCHITECTURE
+fi
 
 if [ "$ALPINE" = "0" ]; then
-    COMPILE_FILE=Dockerfile.bottlecap.compile
+    COMPILE_IMAGE=Dockerfile.bottlecap.compile
 else
     printf "Compiling for alpine\n"
-    COMPILE_FILE=Dockerfile.bottlecap.alpine.compile
+    COMPILE_IMAGE=Dockerfile.bottlecap.alpine.compile
 fi
 
-prepare_folders() {
-    # Move into the root directory, so this script can be called from any directory
-    SCRIPTS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
-    ROOT_DIR=$SCRIPTS_DIR/../..
-    cd $ROOT_DIR
-
-    echo $ROOT_DIR
 
-    BINARY_DIR=".binaries"
-    TARGET_DIR=$(pwd)/$BINARY_DIR
+# Move into the root directory, so this script can be called from any directory
+SCRIPTS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+ROOT_DIR=$SCRIPTS_DIR/../..
+cd $ROOT_DIR
 
-    rm -rf $BINARY_DIR 2>/dev/null
-    mkdir -p $BINARY_DIR
+BINARY_DIR=".binaries"
+TARGET_DIR=$(pwd)/$BINARY_DIR
+BINARY_PATH=$TARGET_DIR/compiled-bottlecap-${SUFFIX}
 
-    cd $ROOT_DIR
-}
+mkdir -p $BINARY_DIR
 
+cd $ROOT_DIR
 
 docker_build() {
     local arch=$1
@@ -56,13 +56,13 @@ docker_build() {
 
     docker buildx build --platform linux/${arch} \
         -t datadog/compile-bottlecap-${SUFFIX} \
-        -f .gitlab/scripts/${file} \
+        -f ./images/${file} \
         --build-arg PLATFORM=$PLATFORM \
-        . -o $TARGET_DIR/compiled-bottlecap-${SUFFIX}
+        . -o $BINARY_PATH
 
-    cp $TARGET_DIR/compiled-bottlecap-${SUFFIX}/bottlecap $TARGET_DIR/bottlecap-${SUFFIX}
+    # Copy the compiled binary to the target directory with the expected name
+    # If it already exist, it will be replaced
+    cp $BINARY_PATH/bottlecap $TARGET_DIR/bottlecap-${SUFFIX}
 }
 
-prepare_folders
-docker_build $ARCHITECTURE $COMPILE_FILE
-
+docker_build $ARCHITECTURE $COMPILE_IMAGE

.gitlab/scripts/compile_go_agent.sh

@@ -38,10 +38,15 @@ fi
 
 
 if [ "$ALPINE" = "0" ]; then
-    COMPILE_FILE=Dockerfile.go_agent.compile
+    COMPILE_IMAGE=Dockerfile.go_agent.compile
 else
     printf "Compiling for alpine\n"
-    COMPILE_FILE=Dockerfile.go_agent.alpine.compile
+    COMPILE_IMAGE=Dockerfile.go_agent.alpine.compile
+fi
+
+if [ -z "$SUFFIX" ]; then
+    printf "No suffix provided, using ${ARCHITECTURE}\n"
+    SUFFIX=$ARCHITECTURE
 fi
 
 # Allow override build tags
@@ -58,9 +63,7 @@ MAIN_DIR=$(pwd) # datadog-lambda-extension
 
 BINARY_DIR=".binaries"
 TARGET_DIR=$MAIN_DIR/$BINARY_DIR
-
-# Make sure the folder does not exist
-rm -rf $BINARY_DIR 2>/dev/null
+BINARY_PATH=$TARGET_DIR/compiled-datadog-agent-${SUFFIX}
 
 mkdir -p $BINARY_DIR
 
@@ -82,14 +85,15 @@ function docker_compile {
 
     docker buildx build --platform linux/${arch} \
         -t datadog/compile-go-agent-${SUFFIX}:${VERSION} \
-        -f ${MAIN_DIR}/.gitlab/scripts/${file} \
+        -f ${MAIN_DIR}/images/${file} \
         --build-arg EXTENSION_VERSION="${VERSION}" \
         --build-arg AGENT_VERSION="${AGENT_VERSION}" \
         --build-arg BUILD_TAGS="${BUILD_TAGS}" \
-        . -o $TARGET_DIR/compiled-datadog-agent-${SUFFIX}
+        . -o $BINARY_PATH
 
-    cp $TARGET_DIR/compiled-datadog-agent-${SUFFIX}/datadog-agent $TARGET_DIR/datadog-agent-${SUFFIX}
+    # Copy the compiled binary to the target directory with the expected name
+    # If it already exist, it will be replaced
+    cp $BINARY_PATH/datadog-agent $TARGET_DIR/datadog-agent-${SUFFIX}
 }
 
-docker_compile $ARCHITECTURE $COMPILE_FILE
-
+docker_compile $ARCHITECTURE $COMPILE_IMAGE

…ipts/Dockerfile.bottlecap.alpine.compile …ages/Dockerfile.bottlecap.alpine.compile.gitlab/scripts/Dockerfile.bottlecap.alpine.compile renamed to images/Dockerfile.bottlecap.alpine.compile .gitlab/scripts/Dockerfile.bottlecap.alpine.compile renamed to images/Dockerfile.bottlecap.alpine.compile

@@ -1,5 +1,7 @@
 FROM alpine:3.16 AS compiler
 ARG PLATFORM
+
+# Install dependencies
 RUN apk add --no-cache curl gcc musl-dev make unzip bash autoconf automake libtool g++
 
 SHELL ["/bin/bash", "-c"]
@@ -13,18 +15,21 @@ RUN curl https://sh.rustup.rs -sSf | \
 ENV PATH=/root/.cargo/bin:$PATH
 RUN rustup component add rust-src --toolchain stable-$PLATFORM-unknown-linux-musl
 
-# Build Bottlecap
+# Copy source code
 RUN mkdir -p /tmp/dd
 COPY ./bottlecap/src /tmp/dd/bottlecap/src
 COPY ./bottlecap/Cargo.toml /tmp/dd/bottlecap/Cargo.toml
 COPY ./bottlecap/Cargo.lock /tmp/dd/bottlecap/Cargo.lock
+
+# Build the binary
+#
 # Added `-C link-arg=-lgcc` for alpine.
 ENV RUSTFLAGS="-C panic=abort -C link-arg=-lgcc"
 WORKDIR /tmp/dd/bottlecap
 RUN --mount=type=cache,target=/root/.cargo/registry cargo +stable build --release --target $PLATFORM-unknown-linux-musl
 RUN cp /tmp/dd/bottlecap/target/$PLATFORM-unknown-linux-musl/release/bottlecap /tmp/dd/bottlecap/bottlecap
 
-# keep the smallest possible docker image
+# Use the smallest image possible
 FROM scratch
 COPY --from=compiler /tmp/dd/bottlecap/bottlecap /
 ENTRYPOINT ["/bottlecap"]

…lab/scripts/Dockerfile.bottlecap.compile images/Dockerfile.bottlecap.compile.gitlab/scripts/Dockerfile.bottlecap.compile renamed to images/Dockerfile.bottlecap.compile .gitlab/scripts/Dockerfile.bottlecap.compile renamed to images/Dockerfile.bottlecap.compile

@@ -1,24 +1,32 @@
 FROM public.ecr.aws/lambda/provided:al2 AS compiler
 ARG PLATFORM
+
+# Install dependencies
 RUN yum install -y curl gcc gcc-c++ make unzip
 
 # Install Protocol Buffers compiler by hand, since AL2 does not have a recent enough version.
 COPY ./scripts/install-protoc.sh /
 RUN chmod +x /install-protoc.sh && /install-protoc.sh
+
+# Install Rust Toolchain
 RUN curl https://sh.rustup.rs -sSf | \
     sh -s -- --profile minimal --default-toolchain stable-$PLATFORM-unknown-linux-gnu -y
 ENV PATH=/root/.cargo/bin:$PATH
 RUN rustup component add rust-src --toolchain stable-$PLATFORM-unknown-linux-gnu
+
+# Copy source code
 RUN mkdir -p /tmp/dd
 COPY ./bottlecap/src /tmp/dd/bottlecap/src
 COPY ./bottlecap/Cargo.toml /tmp/dd/bottlecap/Cargo.toml
 COPY ./bottlecap/Cargo.lock /tmp/dd/bottlecap/Cargo.lock
+
+# Build the binary
 ENV RUSTFLAGS="-C panic=abort"
 WORKDIR /tmp/dd/bottlecap
 RUN --mount=type=cache,target=/usr/local/cargo/registry cargo +stable build --release --target $PLATFORM-unknown-linux-gnu
 RUN cp /tmp/dd/bottlecap/target/$PLATFORM-unknown-linux-gnu/release/bottlecap /tmp/dd/bottlecap/bottlecap
 
-# keep the smallest possible docker image
+# Use smallest image possible
 FROM scratch
 COPY --from=compiler /tmp/dd/bottlecap/bottlecap /
 ENTRYPOINT ["/bottlecap"]

images/Dockerfile.bottlecap.dev

@@ -0,0 +1,19 @@
+FROM ubuntu:22.04 AS compresser
+ARG SUFFIX
+
+RUN apt-get update && apt-get install -y zip binutils
+
+# Copy Go Agent from last release
+COPY --from=public.ecr.aws/datadog/lambda-extension:latest /opt/datadog-agent-go  /datadog-agent-go
+RUN mkdir /extensions
+WORKDIR /extensions
+
+COPY .binaries/bottlecap-${SUFFIX} /extensions/datadog-agent
+COPY ./scripts/datadog_wrapper /datadog_wrapper
+RUN chmod +x /datadog_wrapper
+RUN  zip -r datadog_extension.zip /extensions /datadog_wrapper /datadog-agent-go
+
+# keep the smallest possible docker image
+FROM scratch
+COPY --from=compresser /extensions/datadog_extension.zip /
+ENTRYPOINT ["/datadog_extension.zip"]