Skip to content

Commit 820ddd4

Browse files
philipp94831philipp94831
authored
Merge pull request #35 from bakdata/feature/sam-deploy
Feature/sam deploy
2 parents 0788f5d + e857186 commit 820ddd4

24 files changed

+452
-245
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,4 @@ __pycache__/
66
build/
77
venv/
88
packaged.yaml
9+
.Rhistory

‎.travis.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ matrix:
1818
install:
1919
- PIPENV_VERBOSITY=-1 pipenv install --dev
2020
script:
21-
- ./build.sh $R_VERSION && LOGLEVEL=INFO pipenv run python -m unittest
21+
- ./build.sh ${R_VERSION} && LOGLEVEL=INFO VERSION=${R_VERSION//\./_} pipenv run python -m unittest && if [ "$TRAVIS_BRANCH" = "master" ]; then ./integration_test.sh ${R_VERSION} ; fi
2222
deploy:
2323
- provider: script
2424
script: ./deploy.sh $R_VERSION

‎Pipfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ name = "pypi"
77
awscli = ">=1.16.164"
88

99
[dev-packages]
10-
aws-sam-cli = ">=0.16.1"
10+
aws-sam-cli = ">=0.17.0"
1111

1212
[requires]
1313
python_version = "3.7"

‎Pipfile.lock‎

Lines changed: 44 additions & 42 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎awspack/build.sh‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,16 +12,11 @@ fi
1212

1313
BASE_DIR=$(pwd)
1414
BUILD_DIR=${BASE_DIR}/build/
15-
R_DIR=/opt/R/
1615

1716
rm -rf ${BUILD_DIR}
1817

1918
mkdir -p ${BUILD_DIR}/layer/
2019
docker run -v ${BUILD_DIR}/layer/:/var/awspack -v ${BASE_DIR}/entrypoint.sh:/entrypoint.sh \
2120
lambda-r:build-${VERSION} /entrypoint.sh
2221
sudo chown -R $(whoami):$(whoami) ${BUILD_DIR}/layer/
23-
cd ${BUILD_DIR}/layer/
24-
chmod -R 755 .
25-
zip -r -q awspack-${VERSION}.zip .
26-
mkdir -p ${BUILD_DIR}/dist/
27-
mv awspack-${VERSION}.zip ${BUILD_DIR}/dist/
22+
chmod -R 755 ${BUILD_DIR}/layer/

‎awspack/compile.sh‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,4 @@ rm -rf ${BUILD_DIR}
1919
export R_LIBS=${BUILD_DIR}/layer/R/library
2020
mkdir -p ${R_LIBS}
2121
${R_DIR}/bin/Rscript -e 'install.packages("awspack", repos="http://cran.r-project.org")'
22-
cd ${BUILD_DIR}/layer/
23-
chmod -R 755 .
24-
zip -r -q awspack-${VERSION}.zip .
25-
mkdir -p ${BUILD_DIR}/dist/
26-
mv awspack-${VERSION}.zip ${BUILD_DIR}/dist/
22+
chmod -R 755 ${BUILD_DIR}/layer/

‎awspack/deploy.sh‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,14 @@ else
1010
VERSION=$1
1111
fi
1212

13-
./build.sh ${VERSION}
13+
BASE_DIR=$(pwd)
14+
BUILD_DIR=${BASE_DIR}/build/
15+
16+
cd ${BUILD_DIR}/layer/
17+
zip -r -q awspack-${VERSION}.zip .
18+
mkdir -p ${BUILD_DIR}/dist/
19+
mv awspack-${VERSION}.zip ${BUILD_DIR}/dist/
20+
version_="${VERSION//\./_}"
1421
aws lambda publish-layer-version \
15-
--layer-name r-awspack-${VERSION} \
16-
--zip-file fileb://build/dist/awspack-${VERSION}.zip
22+
--layer-name r-awspack-${version_} \
23+
--zip-file fileb://${BUILD_DIR}/dist/awspack-${VERSION}.zip

‎build.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@ BASE_DIR=$(pwd)
1515
cd ${BASE_DIR}/r
1616
./build.sh ${VERSION}
1717
cd ${BASE_DIR}/runtime
18-
./build.sh ${VERSION}
18+
./build.sh
1919
cd ${BASE_DIR}/recommended
20-
./build.sh ${VERSION}
20+
./build.sh
2121
cd ${BASE_DIR}/awspack
2222
./build.sh ${VERSION}

‎deploy.sh‎

Lines changed: 46 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -13,47 +13,56 @@ fi
1313
function releaseToRegion {
1414
version=$1
1515
region=$2
16-
layer=$3
1716
bucket="aws-lambda-r-runtime.$region"
18-
resource="R-$version/$layer-$version.zip"
19-
layer_name="r-$layer-$version"
20-
layer_name="${layer_name//\./_}"
21-
echo "publishing layer $layer_name to region $region"
22-
aws s3 cp ${layer}/build/dist/${layer}-${version}.zip s3://${bucket}/${resource} --region ${region}
23-
response=$(aws lambda publish-layer-version \
24-
--layer-name ${layer_name} \
25-
--content S3Bucket=${bucket},S3Key=${resource} \
26-
--license-info MIT \
27-
--region ${region})
28-
version_number=$(jq -r '.Version' <<< "$response")
29-
aws lambda add-layer-version-permission \
30-
--layer-name ${layer_name} \
31-
--version-number ${version_number} \
32-
--principal "*" \
33-
--statement-id publish \
34-
--action lambda:GetLayerVersion \
17+
echo "publishing layers to region $region"
18+
sam package \
19+
--output-template-file packaged.yaml \
20+
--s3-bucket ${bucket} \
3521
--region ${region}
36-
layer_arn=$(jq -r '.LayerVersionArn' <<< "$response")
37-
echo "published layer $layer_arn"
22+
version_="${version//\./_}"
23+
stack_name=r-${version//\./-}
24+
sam deploy \
25+
--template-file packaged.yaml \
26+
--stack-name ${stack_name} \
27+
--parameter-overrides Version=${version_} \
28+
--no-fail-on-empty-changeset \
29+
--region ${region}
30+
layers=(runtime recommended awspack)
31+
for layer in "${layers[@]}"
32+
do
33+
layer_output=${layer}Layer
34+
layer_arn=$(aws cloudformation describe-stacks \
35+
--stack-name ${stack_name} \
36+
--query "Stacks[0].Outputs[?OutputKey=='$layer_output'].OutputValue" \
37+
--output text \
38+
--region ${region})
39+
layer_name=${layer_arn%:*}
40+
version_number=${layer_arn##*:}
41+
aws lambda add-layer-version-permission \
42+
--layer-name ${layer_name} \
43+
--version-number ${version_number} \
44+
--principal "*" \
45+
--statement-id publish \
46+
--action lambda:GetLayerVersion \
47+
--region ${region}
48+
echo "published layer $layer_arn"
49+
done
3850
}
3951

40-
regions=(us-east-1 us-east-2
41-
us-west-1 us-west-2
42-
ap-south-1
43-
ap-northeast-1 ap-northeast-2
44-
ap-southeast-1 ap-southeast-2
45-
ca-central-1
46-
eu-central-1
47-
eu-north-1
48-
eu-west-1 eu-west-2 eu-west-3
49-
sa-east-1)
52+
regions=(
53+
us-east-1 us-east-2
54+
us-west-1 us-west-2
55+
ap-south-1
56+
ap-northeast-1 ap-northeast-2
57+
ap-southeast-1 ap-southeast-2
58+
ca-central-1
59+
eu-central-1
60+
eu-north-1
61+
eu-west-1 eu-west-2 eu-west-3
62+
sa-east-1
63+
)
5064

51-
layers=(runtime recommended awspack)
52-
53-
for layer in "${layers[@]}"
65+
for region in "${regions[@]}"
5466
do
55-
for region in "${regions[@]}"
56-
do
57-
releaseToRegion ${VERSION} ${region} ${layer}
58-
done
67+
releaseToRegion ${VERSION} ${region}
5968
done

‎integration_test.sh‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
if [[ -z ${1+x} ]];
6+
then
7+
echo 'version number required'
8+
exit 1
9+
else
10+
R_VERSION=$1
11+
fi
12+
13+
function integrationTest {
14+
version=$1
15+
region=$2
16+
bucket="aws-lambda-r-runtime.$region"
17+
echo "Integration testing in region $region"
18+
sam package \
19+
--output-template-file packaged.yaml \
20+
--s3-bucket ${bucket} \
21+
--template-file test-template.yaml \
22+
--region ${region}
23+
version_="${version//\./_}"
24+
stack_name=r-${version//\./-}-test
25+
sam deploy \
26+
--template-file packaged.yaml \
27+
--stack-name ${stack_name} \
28+
--capabilities CAPABILITY_IAM \
29+
--parameter-overrides Version=${version_} \
30+
--no-fail-on-empty-changeset \
31+
--region ${region}
32+
VERSION=${version_} INTEGRATION_TEST=True AWS_DEFAULT_REGION=${region} pipenv run python -m unittest
33+
}
34+
35+
regions=(
36+
us-east-1
37+
)
38+
39+
for region in "${regions[@]}"
40+
do
41+
integrationTest ${R_VERSION} ${region}
42+
done

0 commit comments

Comments
 (0)