Merge pull request #287 from nextstrain/aws-v2

Update AWS CLI to v2

Author: joverlee521

Dockerfile

@@ -28,6 +28,7 @@ COPY --from=xx / /
 # make: used for building from Makefiles (search for usage); may be used by package managers to build from source
 # nodejs: for installing Auspice
 # clang: for compiling C/C++ projects; may be used by package managers to build from source
+# unzip: for AWS CLI
 RUN apt-get update && apt-get install -y --no-install-recommends \
         autoconf \
         automake \
@@ -36,7 +37,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         curl \
         git \
         make \
-        dpkg-dev
+        dpkg-dev \
+        unzip
 
 # Install a specific Node.js version
 # https://github.com/nodesource/distributions/blob/ba48c1fe6e843e9ffb60aefc5da5d00234c83f04/README.md#using-debian-as-root-nodejs-20
@@ -208,6 +210,14 @@ RUN curl -fsSL https://ftp.ncbi.nlm.nih.gov/pub/datasets/command-line/v2/linux-$
 RUN curl -fsSL https://ftp.ncbi.nlm.nih.gov/pub/datasets/command-line/v2/linux-${TARGETARCH}/dataformat \
   -o /final/bin/dataformat
 
+# Install tooling for our AWS Batch builds, which use `aws s3`.
+# Using `xx-info march` to translate amd64 -> x86_64 and arm64 -> aarch64
+# based on https://github.com/docker/buildx/discussions/2001
+RUN XX_MARCH=$(xx-info march) \
+  && curl -fsSL --proto '=https' "https://awscli.amazonaws.com/awscli-exe-linux-${XX_MARCH}.zip" -o "awscliv2.zip" \
+  && unzip awscliv2.zip "aws/dist/*" -d /final/libexec/aws-cli \
+  && ln -srv /final/libexec/aws-cli/aws/dist/aws /final/bin/aws
+
 # ———————————————————————————————————————————————————————————————————— #
 
 # Define a builder stage that runs on the target platform.
@@ -244,11 +254,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 # Install envdir, which is used by pathogen builds
 RUN pip3 install envdir==1.0.1
 
-# Install tooling for our AWS Batch builds, which use `aws s3`.
-RUN python3 -m venv /usr/local/libexec/awscli \
- && /usr/local/libexec/awscli/bin/python -m pip install awscli==1.18.195 \
- && ln -sv /usr/local/libexec/awscli/bin/aws /usr/local/bin/aws
-
 # Install Snakemake and related optional dependencies.
 # Pinned to 9.17.0 for stability, latest guaranteed compatible version
 # based on <https://github.com/nextstrain/public/issues/39>
@@ -389,9 +394,6 @@ RUN chmod a+rx /usr/local/bin/* /usr/local/libexec/*
 # Add installed Python libs
 COPY --from=builder-target-platform /usr/local/lib/python3.11/site-packages/ /usr/local/lib/python3.11/site-packages/
 
-# AWS CLI
-COPY --from=builder-target-platform /usr/local/libexec/awscli/ /usr/local/libexec/awscli/
-
 # Add installed Python scripts that we need.
 #
 # XXX TODO: This isn't great.  It's prone to needing manual updates because it
@@ -403,7 +405,6 @@ COPY --from=builder-target-platform /usr/local/libexec/awscli/ /usr/local/libexe
 #   -trs, 15 June 2018
 COPY --from=builder-target-platform \
     /usr/local/bin/augur \
-    /usr/local/bin/aws \
     /usr/local/bin/bio \
     /usr/local/bin/envdir \
     /usr/local/bin/evofr \

tests/envdir-url.t

@@ -24,7 +24,7 @@ Files are populated from NEXTSTRAIN_ENVD_URL.
 
   $ aws s3 cp --quiet env.d.zip "$NEXTSTRAIN_ENVD_URL"
 
-  $ docker run --rm -e NEXTSTRAIN_ENVD_URL --env=AWS_{ACCESS_KEY_ID,SECRET_ACCESS_KEY,SESSION_TOKEN} "$IMAGE" \
+  $ docker run --rm -e NEXTSTRAIN_ENVD_URL --env=AWS_{ACCESS_KEY_ID,SECRET_ACCESS_KEY,SESSION_TOKEN,DEFAULT_REGION} "$IMAGE" \
   >   bash -eu -c 'echo "$a"; echo "$b"'
   download: s3://nextstrain-tmp/*.zip to ../env.d.zip (glob)
   Archive:  /nextstrain/env.d.zip
@@ -37,7 +37,7 @@ Files are populated from NEXTSTRAIN_ENVD_URL.
 
 Files and NEXTSTRAIN_ENVD_URL are removed with NEXTSTRAIN_DELETE_ENVD=1.
 
-  $ docker run --rm -e NEXTSTRAIN_DELETE_ENVD=1 -e NEXTSTRAIN_ENVD_URL --env=AWS_{ACCESS_KEY_ID,SECRET_ACCESS_KEY,SESSION_TOKEN} "$IMAGE" \
+  $ docker run --rm -e NEXTSTRAIN_DELETE_ENVD=1 -e NEXTSTRAIN_ENVD_URL --env=AWS_{ACCESS_KEY_ID,SECRET_ACCESS_KEY,SESSION_TOKEN,DEFAULT_REGION} "$IMAGE" \
   >   bash -eu -c 'echo "$a"; echo "$b"; ls -1 /nextstrain/env.d'
   download: s3://nextstrain-tmp/*.zip to ../env.d.zip (glob)
   Archive:  /nextstrain/env.d.zip