Skip to content

Commit caf6ac9

Browse files
roger-zhanggroger-zhangg
authored
Add binary integration testing support to GitHub Actions workflow (#8693)
* adds binary test for linux * gh token * test * latest release * nit * unset * nit * nit * nit * zig * nit * address feedback * use release instead * upload different test type report to its own folder * move typical test sam cli into venv * still use system * nit * nit * nit
1 parent abe5d8b commit caf6ac9

6 files changed

Lines changed: 169 additions & 32 deletions

File tree

‎.github/workflows/build.yml‎

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -186,18 +186,9 @@ jobs:
186186
with:
187187
dotnet-version: '10.0.x'
188188
# Install and configure Rust & Cargo Lambda
189-
- name: Install and configure Rust & Cargo Lambda
189+
- name: Install Rust toolchain and cargo-lambda
190190
if: ${{ matrix.os == 'ubuntu-latest' }}
191-
run: |
192-
: install rustup if needed
193-
if ! command -v rustup &> /dev/null ; then
194-
curl --proto '=https' --tlsv1.2 --retry 10 --retry-connrefused -fsSL "https://sh.rustup.rs" | sh -s -- --default-toolchain none -y
195-
echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> $GITHUB_PATH
196-
fi
197-
rustup toolchain install stable --profile minimal --no-self-update
198-
rustup default stable
199-
pip install "cargo-lambda==$CARGO_LAMBDA_VERSION"
200-
echo "$HOME/.local/bin" >> $GITHUB_PATH
191+
run: bash tests/install-rust.sh
201192
- name: Init samdev
202193
run: make init
203194
- name: uv install setuptools in Python3.12

‎.github/workflows/integration-tests.yml‎

Lines changed: 56 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,23 @@ on:
55
# Run at 7pm PST (3am UTC next day) Monday-Friday
66
# This translates to Tuesday-Saturday 3am UTC
77
- cron: '0 3 * * 2-6'
8+
release:
9+
types: [published, edited]
10+
# published → latest-release binary test (full releases only)
11+
# edited + prerelease → nightly binary test (nightly pre-release gets edited with new artifacts)
812
workflow_dispatch: # Allow manual triggering
913
# NOTE: This workflow can only be manually triggered from develop or main branches
1014
# The other branch will not pass the OIDC permission check
15+
inputs:
16+
install_mode:
17+
description: '"code" to install from source (make init), "nightly-release" to install SAM CLI nightly binary, "latest-release" to install latest stable release binary'
18+
required: false
19+
default: 'code'
20+
type: choice
21+
options:
22+
- code
23+
- nightly-release
24+
- latest-release
1125

1226
permissions:
1327
id-token: write # Required for OIDC
@@ -153,20 +167,9 @@ jobs:
153167
with:
154168
ruby-version: '3.4.7'
155169

156-
- name: Install Rust toolchain
170+
- name: Install Rust toolchain and cargo-lambda
157171
if: contains(fromJSON('["build-x86-1", "build-x86-2", "build-arm64", "cloud-based-tests", "tier1-finch"]'), matrix.test_suite)
158-
run: |
159-
curl --proto '=https' --tlsv1.2 --retry 10 --retry-connrefused -fsSL https://sh.rustup.rs | sh -s -- --default-toolchain none -y
160-
source $HOME/.cargo/env
161-
rustup toolchain install stable --profile minimal --no-self-update
162-
rustup default stable
163-
rustup target add x86_64-unknown-linux-gnu --toolchain stable
164-
rustup target add aarch64-unknown-linux-gnu --toolchain stable
165-
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
166-
167-
- name: Install cargo-lambda
168-
if: contains(fromJSON('["build-x86-1", "build-x86-2", "build-arm64", "cloud-based-tests", "tier1-finch"]'), matrix.test_suite)
169-
run: pip install cargo-lambda==${{ env.CARGO_LAMBDA_VERSION }}
172+
run: bash tests/install-rust.sh
170173

171174
- name: Install Terraform
172175
if: contains(fromJSON('["terraform-build", "terraform-start-api", "terraform-invoke-start-lambda", "cloud-based-tests", "tier1-finch"]'), matrix.test_suite)
@@ -193,18 +196,52 @@ jobs:
193196
if: "!contains(fromJSON('[\"build-x86-1\", \"build-x86-2\", \"build-x86-container-1\", \"build-x86-container-2\", \"tier1-finch\"]'), matrix.test_suite)"
194197
run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
195198

196-
- name: Initialize project
199+
- name: Determine test type
197200
run: |
198-
if [[ "${{ matrix.test_suite }}" == "tier1-finch" ]]; then
199-
export CONTAINER_RUNTIME=finch
201+
echo "CONTAINER_RUNTIME=${{ matrix.test_suite == 'tier1-finch' && 'finch' || 'docker' }}" >> $GITHUB_ENV
202+
if [[ "${{ inputs.install_mode }}" == "nightly-release" ]] || \
203+
[[ "${{ github.event.action }}" == "edited" && "${{ github.event.release.prerelease }}" == "true" ]]; then
204+
echo "TEST_TYPE=nightly-release" >> $GITHUB_ENV
205+
elif [[ "${{ inputs.install_mode }}" == "latest-release" ]] || \
206+
[[ "${{ github.event.action }}" == "published" && "${{ github.event.release.prerelease }}" != "true" ]]; then
207+
echo "TEST_TYPE=latest-release" >> $GITHUB_ENV
208+
elif [[ "${{ github.ref }}" == "refs/heads/master" ]]; then
209+
echo "TEST_TYPE=master" >> $GITHUB_ENV
210+
elif [[ "${{ github.event_name }}" == "schedule" ]] || [[ "${{ github.ref }}" == "refs/heads/develop" ]]; then
211+
echo "TEST_TYPE=develop" >> $GITHUB_ENV
200212
else
201-
export CONTAINER_RUNTIME=docker
213+
echo "TEST_TYPE=other" >> $GITHUB_ENV
202214
fi
215+
216+
- name: Initialize integration test
217+
if: env.TEST_TYPE == 'develop' || env.TEST_TYPE == 'master' || env.TEST_TYPE == 'other'
218+
run: |
203219
make init
220+
echo "SCRIPT_PY=python3.11" >> $GITHUB_ENV
221+
222+
- name: Initialize nightly release test
223+
if: env.TEST_TYPE == 'nightly-release'
224+
env:
225+
GH_TOKEN: ${{ github.token }}
226+
run: |
227+
make setup-pytest
228+
make init-nightly
229+
echo "SAM_CLI_DEV=" >> $GITHUB_ENV
230+
echo "SCRIPT_PY=$HOME/pytest/bin/python3" >> $GITHUB_ENV
231+
232+
- name: Initialize latest release test
233+
if: env.TEST_TYPE == 'latest-release'
234+
env:
235+
GH_TOKEN: ${{ github.token }}
236+
run: |
237+
make setup-pytest
238+
make init-latest-release
239+
echo "SAM_CLI_DEV=" >> $GITHUB_ENV
240+
echo "SCRIPT_PY=$HOME/pytest/bin/python3" >> $GITHUB_ENV
204241
205242
- name: Get testing resources and credentials
206243
if: "!contains(fromJSON('[\"build-x86-1\", \"build-x86-2\", \"build-arm64\", \"build-x86-container-1\", \"build-x86-container-2\", \"build-arm64-container-1\", \"build-arm64-container-2\", \"local-invoke\", \"local-start-api\", \"local-start-lambda\"]'), matrix.test_suite)"
207-
run: python3.11 tests/setup_testing_resources.py
244+
run: $SCRIPT_PY tests/setup_testing_resources.py
208245

209246
- name: Login to Public ECR
210247
run: |
@@ -398,4 +435,4 @@ jobs:
398435
if: always()
399436
env:
400437
TESTREPORTING_S3: ${{ secrets.TESTREPORTING_S3 }}
401-
run: python3.11 tests/reset_testing_resources.py ${{ matrix.test_suite }}
438+
run: $SCRIPT_PY tests/reset_testing_resources.py ${{ matrix.test_suite }}

‎Makefile‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# environment variable.
33
SAM_CLI_TELEMETRY ?= 0
44

5-
.PHONY: schema
5+
.PHONY: schema init-nightly init-latest-release setup-pytest
66

77
# Initialize environment specifically for Github action tests using uv
88
init:
@@ -13,6 +13,21 @@ init:
1313
SAM_CLI_DEV=1 pip install -e '.[dev]'; \
1414
fi
1515

16+
# Set up a pytest venv with test dependencies
17+
setup-pytest:
18+
python3.11 -m venv $(HOME)/pytest
19+
uv pip install --python $(HOME)/pytest/bin/python3 -r requirements/dev.txt -r requirements/base.txt
20+
sudo ln -sf $(HOME)/pytest/bin/pytest /usr/local/bin/pytest
21+
pytest --version
22+
23+
# Install SAM CLI nightly binary
24+
init-nightly:
25+
bash tests/install-sam-cli-binary.sh sam-cli-nightly
26+
27+
# Install SAM CLI latest release binary
28+
init-latest-release:
29+
bash tests/install-sam-cli-binary.sh
30+
1631
test:
1732
# Run unit tests and fail if coverage falls below 94%
1833
pytest --cov samcli --cov schema --cov-report term-missing --cov-fail-under 94 tests/unit

‎tests/install-rust.sh‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/bin/bash
2+
# Install Rust toolchain and cargo-lambda for SAM CLI integration tests.
3+
# Usage: ./tests/install-rust.sh [CARGO_LAMBDA_VERSION]
4+
# CARGO_LAMBDA_VERSION defaults to env var or "v0.17.1"
5+
set -euo pipefail
6+
7+
CARGO_LAMBDA_VERSION="${1:-${CARGO_LAMBDA_VERSION:-v0.17.1}}"
8+
9+
# Install rustup if not present
10+
if ! command -v rustup &> /dev/null; then
11+
curl --proto '=https' --tlsv1.2 --retry 10 --retry-connrefused -fsSL https://sh.rustup.rs | sh -s -- --default-toolchain none -y
12+
source "$HOME/.cargo/env"
13+
if [ -n "${GITHUB_PATH:-}" ]; then
14+
echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> "$GITHUB_PATH"
15+
fi
16+
fi
17+
18+
rustup toolchain install stable --profile minimal --no-self-update
19+
rustup default stable
20+
rustup target add x86_64-unknown-linux-gnu --toolchain stable
21+
rustup target add aarch64-unknown-linux-gnu --toolchain stable
22+
23+
# Install cargo-lambda and ziglang
24+
python3.11 -m pip install "cargo-lambda==$CARGO_LAMBDA_VERSION"
25+
26+
# Create a zig wrapper so SAM CLI's cargo-lambda can find it
27+
printf '#!/bin/bash\nexec python3.11 -m ziglang "$@"\n' | sudo tee /usr/local/bin/zig > /dev/null
28+
sudo chmod +x /usr/local/bin/zig
29+
30+
rustc -V
31+
cargo -V
32+
zig version

‎tests/install-sam-cli-binary.sh‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
#!/bin/bash
2+
# Install SAM CLI binary from a GitHub release, auto-detecting OS and architecture.
3+
# Usage: ./test/install-sam-cli-binary.sh [<tag>]
4+
# If <tag> is provided (e.g. sam-cli-nightly), downloads from that release.
5+
# If omitted, downloads from the latest (non-pre-release) release.
6+
set -euo pipefail
7+
8+
TAG="${1:-}"
9+
OS="$(uname -s | tr '[:upper:]' '[:lower:]')"
10+
ARCH="$(uname -m)"
11+
12+
# Map to asset naming convention
13+
case "$OS" in
14+
linux)
15+
case "$ARCH" in
16+
x86_64) ASSET="aws-sam-cli-linux-x86_64.zip" ;;
17+
aarch64) ASSET="aws-sam-cli-linux-arm64.zip" ;;
18+
*) echo "Unsupported Linux architecture: $ARCH"; exit 1 ;;
19+
esac
20+
;;
21+
darwin)
22+
case "$ARCH" in
23+
x86_64) ASSET="aws-sam-cli-macos-x86_64.pkg" ;;
24+
arm64) ASSET="aws-sam-cli-macos-arm64.pkg" ;;
25+
*) echo "Unsupported macOS architecture: $ARCH"; exit 1 ;;
26+
esac
27+
;;
28+
*)
29+
echo "Unsupported OS: $OS"; exit 1
30+
;;
31+
esac
32+
33+
echo "Detected OS=$OS ARCH=$ARCH -> downloading $ASSET"
34+
35+
# Download to a temp directory and clean up on exit
36+
TMPDIR="$(mktemp -d)"
37+
trap 'rm -rf "$TMPDIR"' EXIT
38+
39+
if [ -n "$TAG" ]; then
40+
gh release download "$TAG" --repo aws/aws-sam-cli --pattern "$ASSET" --clobber --dir "$TMPDIR"
41+
else
42+
gh release download --repo aws/aws-sam-cli --pattern "$ASSET" --clobber --dir "$TMPDIR"
43+
fi
44+
45+
# Install
46+
case "$ASSET" in
47+
*.zip)
48+
unzip -o "$TMPDIR/$ASSET" -d "$TMPDIR/sam-installation"
49+
sudo "$TMPDIR/sam-installation/install" --update
50+
;;
51+
*.pkg)
52+
sudo installer -pkg "$TMPDIR/$ASSET" -target /
53+
;;
54+
esac
55+
56+
# Nightly installs as sam-nightly; rename to sam
57+
if [ -f /usr/local/bin/sam-nightly ]; then
58+
sudo mv /usr/local/bin/sam-nightly /usr/local/bin/sam
59+
fi
60+
61+
sam --version

‎tests/reset_testing_resources.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,8 @@ def upload_test_reports(test_suite):
5757
s3_client = boto3.client("s3", config=DEFAULT_BOTO_CONFIG)
5858

5959
date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d")
60-
s3_prefix = f"github/{date_str}/{run_id}"
60+
test_type = os.environ.get("TEST_TYPE", "unknown")
61+
s3_prefix = f"github/{test_type}/{date_str}/{run_id}"
6162

6263
for report_path in reports:
6364
s3_key = f"{s3_prefix}/{report_path}"

0 commit comments

Comments
 (0)