Skip to content

Commit ba7d019

Browse files
anth-volkanth-volk
authored
Merge pull request #843 from PolicyEngine/fix/improve-docker-pipeline
Improve Docker pipeline
2 parents ac4af50 + fe0d8d0 commit ba7d019

15 files changed

Lines changed: 620 additions & 126 deletions

‎.dockerignore‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
# Git
2+
.git
3+
.gitignore
4+
5+
# Python
6+
__pycache__
7+
*.pyc
8+
*.pyo
9+
*.pyd
10+
.Python
11+
env
12+
pip-log.txt
13+
pip-delete-this-directory.txt
14+
.tox
15+
.coverage
16+
.coverage.*
17+
.cache
18+
nosetests.xml
19+
coverage.xml
20+
*.cover
21+
*.log
22+
.git
23+
.mypy_cache
24+
.pytest_cache
25+
.hypothesis
26+
27+
# Virtual environments
28+
.env
29+
.venv
30+
env/
31+
venv/
32+
ENV/
33+
env.bak/
34+
venv.bak/
35+
36+
# IDE
37+
.vscode/
38+
.idea/
39+
*.swp
40+
*.swo
41+
*~
42+
43+
# OS
44+
.DS_Store
45+
.DS_Store?
46+
._*
47+
.Spotlight-V100
48+
.Trashes
49+
ehthumbs.db
50+
Thumbs.db
51+
52+
# Documentation
53+
docs/
54+
*.md
55+
README*
56+
57+
# Tests
58+
tests/
59+
test_*
60+
*_test.py
61+
62+
# Development
63+
under_development/
64+
gcp/
65+
Dockerfile*
66+
.dockerignore
67+
68+
# CI/CD
69+
.github/
70+
.github/workflows/
71+
72+
# Logs
73+
logs/
74+
*.log
75+
76+
# Temporary files
77+
tmp/
78+
temp/

‎.github/scripts/deploy-app-engine.sh‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
#!/bin/bash
2+
3+
# Deploy to Google Cloud App Engine using pre-built Docker image
4+
set -e
5+
6+
# Get required environment variables
7+
IMAGE_NAME="${IMAGE_NAME}"
8+
IMAGE_TAG="${IMAGE_TAG}"
9+
SERVICE_ACCOUNT="${SERVICE_ACCOUNT}"
10+
APP_YAML_PATH="${APP_YAML_PATH}"
11+
12+
if [ -z "$IMAGE_NAME" ] || [ -z "$IMAGE_TAG" ] || [ -z "$SERVICE_ACCOUNT" ] || [ -z "$APP_YAML_PATH" ]; then
13+
echo "Error: Required environment variables not set"
14+
echo "IMAGE_NAME: $IMAGE_NAME"
15+
echo "IMAGE_TAG: $IMAGE_TAG"
16+
echo "SERVICE_ACCOUNT: $SERVICE_ACCOUNT"
17+
echo "APP_YAML_PATH: $APP_YAML_PATH"
18+
exit 1
19+
fi
20+
21+
echo "Deploying pre-built Docker image to App Engine..."
22+
echo "Image: $IMAGE_NAME:$IMAGE_TAG"
23+
echo "Version: $IMAGE_TAG"
24+
echo "Service Account: $SERVICE_ACCOUNT"
25+
echo "App YAML: $APP_YAML_PATH"
26+
# Define environment variables to set
27+
declare -A ENV_VARS=(
28+
["AUTH0_ADDRESS_NO_DOMAIN"]="$AUTH0_ADDRESS_NO_DOMAIN"
29+
["AUTH0_AUDIENCE_NO_DOMAIN"]="$AUTH0_AUDIENCE_NO_DOMAIN"
30+
["USER_ANALYTICS_DB_USERNAME"]="$USER_ANALYTICS_DB_USERNAME"
31+
["USER_ANALYTICS_DB_PASSWORD"]="$USER_ANALYTICS_DB_PASSWORD"
32+
["USER_ANALYTICS_DB_CONNECTION_NAME"]="$USER_ANALYTICS_DB_CONNECTION_NAME"
33+
["ANTHROPIC_API_KEY"]="$ANTHROPIC_API_KEY"
34+
)
35+
36+
# Build the --set-env-vars string
37+
ENV_VARS_STRING=""
38+
for key in "${!ENV_VARS[@]}"; do
39+
if [ -n "${ENV_VARS[$key]}" ]; then
40+
ENV_VARS_STRING="$ENV_VARS_STRING --set-env-vars $key=${ENV_VARS[$key]}"
41+
else
42+
echo "Warning: $key is not set"
43+
fi
44+
done
45+
46+
echo "Environment Variables: ${#ENV_VARS[@]} variables will be set"
47+
48+
# Deploy to App Engine using the pre-built image
49+
gcloud app deploy "$APP_YAML_PATH" \
50+
--image-url="$IMAGE_NAME:$IMAGE_TAG" \
51+
--version="$IMAGE_TAG" \
52+
--service-account="$SERVICE_ACCOUNT" \
53+
--quiet \
54+
$ENV_VARS_STRING
55+
56+
echo "App Engine deployment completed successfully"

‎.github/scripts/set-traffic.sh‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/bin/bash
2+
3+
# Set traffic to new App Engine version
4+
set -e
5+
6+
# Get required environment variables
7+
SERVICE_NAME="${SERVICE_NAME}"
8+
VERSION="${VERSION}"
9+
10+
if [ -z "$SERVICE_NAME" ] || [ -z "$VERSION" ]; then
11+
echo "Error: SERVICE_NAME and VERSION environment variables must be set"
12+
exit 1
13+
fi
14+
15+
echo "Setting traffic to new App Engine version..."
16+
echo "Service: $SERVICE_NAME"
17+
echo "Version: $VERSION"
18+
19+
# Set traffic to the new version
20+
gcloud app services set-traffic "$SERVICE_NAME" \
21+
--splits="$VERSION=1.0"
22+
23+
echo "Traffic successfully set to version $VERSION"

‎.github/scripts/verify-deployment.sh‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/bin/bash
2+
3+
# Verify App Engine deployment
4+
set -e
5+
6+
# Get required environment variables
7+
SERVICE_NAME="${SERVICE_NAME}"
8+
9+
if [ -z "$SERVICE_NAME" ]; then
10+
echo "Error: SERVICE_NAME environment variable must be set"
11+
exit 1
12+
fi
13+
14+
echo "Verifying App Engine deployment..."
15+
echo "Service: $SERVICE_NAME"
16+
17+
# List recent versions to verify deployment
18+
gcloud app versions list \
19+
--service="$SERVICE_NAME" \
20+
--limit=5
21+
22+
echo "Deployment verification completed"

‎.github/scripts/verify-image.sh‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/bin/bash
2+
3+
# Verify Docker image exists in GitHub Container Registry
4+
set -e
5+
6+
# Get image name and tag from environment variables
7+
IMAGE_NAME="${IMAGE_NAME}"
8+
IMAGE_TAG="${IMAGE_TAG}"
9+
10+
if [ -z "$IMAGE_NAME" ] || [ -z "$IMAGE_TAG" ]; then
11+
echo "Error: IMAGE_NAME and IMAGE_TAG environment variables must be set"
12+
exit 1
13+
fi
14+
15+
echo "Verifying image exists in GitHub Container Registry..."
16+
echo "Image: $IMAGE_NAME:$IMAGE_TAG"
17+
18+
# Pull the image to verify it exists
19+
docker pull "$IMAGE_NAME:$IMAGE_TAG"
20+
21+
echo "Image verified and ready for deployment"

‎.github/workflows/deploy-production.yml‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
name: 'Deployment step 2: Deploy to Production'
2+
3+
on:
4+
push:
5+
branches: [ main ]
6+
7+
env:
8+
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
9+
REGION: us-central1
10+
SERVICE_NAME: policyengine-household-api
11+
IMAGE_NAME: ghcr.io/policyengine-household-api/policyengine-household-api
12+
PYTHON_VERSION: '3.12'
13+
14+
jobs:
15+
lint-and-test:
16+
name: Lint and test
17+
runs-on: ubuntu-latest
18+
if: |
19+
(github.repository == 'PolicyEngine/policyengine-household-api')
20+
&& (github.event.head_commit.message == 'Update PolicyEngine Household API')
21+
steps:
22+
- name: Checkout code
23+
uses: actions/checkout@v4
24+
25+
- name: Set up Python
26+
uses: actions/setup-python@v4
27+
with:
28+
python-version: ${{ env.PYTHON_VERSION }}
29+
30+
- name: Install dependencies
31+
run: make install
32+
33+
- name: Run linter
34+
run: make format
35+
36+
- name: Run tests
37+
run: make test
38+
39+
build-docker:
40+
name: Build Docker image
41+
runs-on: ubuntu-latest
42+
if: |
43+
(github.repository == 'PolicyEngine/policyengine-household-api')
44+
&& (github.event.head_commit.message == 'Update PolicyEngine Household API')
45+
needs: [lint-and-test]
46+
steps:
47+
- name: Checkout code
48+
uses: actions/checkout@v4
49+
50+
- name: Set up Docker Buildx
51+
uses: docker/setup-buildx-action@v3
52+
53+
- name: Log in to GitHub Container Registry
54+
uses: docker/login-action@v2
55+
with:
56+
registry: ghcr.io
57+
username: ${{ github.actor }}
58+
password: ${{ secrets.GITHUB_TOKEN }}
59+
60+
- name: Extract metadata
61+
id: meta
62+
uses: docker/metadata-action@v5
63+
with:
64+
images: ${{ env.IMAGE_NAME }}
65+
tags: |
66+
type=sha,prefix={{branch}}-
67+
type=raw,value=latest,enable={{is_default_branch}}
68+
type=raw,value=python${{ env.PYTHON_VERSION }},enable={{is_default_branch}}
69+
70+
- name: Build and push Docker image
71+
uses: docker/build-push-action@v5
72+
with:
73+
context: .
74+
file: ./dockerfiles/production/Dockerfile.production
75+
push: true
76+
tags: ${{ steps.meta.outputs.tags }}
77+
labels: ${{ steps.meta.outputs.labels }}
78+
cache-from: type=gha
79+
cache-to: type=gha,mode=max
80+
81+
- name: Verify image was pushed
82+
run: |
83+
echo "Verifying image was pushed to GitHub Container Registry..."
84+
docker pull ${{ env.IMAGE_NAME }}:${{ github.sha }}
85+
echo "Image successfully pushed and can be pulled"
86+
87+
# Deploy to App Engine using pre-built Docker image from GitHub Container Registry
88+
deploy:
89+
runs-on: ubuntu-latest
90+
if: |
91+
(github.repository == 'PolicyEngine/policyengine-household-api')
92+
&& (github.event.head_commit.message == 'Update PolicyEngine Household API')
93+
needs: [lint-and-test, build-docker]
94+
steps:
95+
- name: Checkout code
96+
uses: actions/checkout@v4
97+
98+
- name: Authenticate to Google Cloud
99+
uses: google-github-actions/auth@v2
100+
with:
101+
credentials_json: ${{ secrets.GCP_SA_KEY }}
102+
103+
- name: Log in to GitHub Container Registry for image verification
104+
uses: docker/login-action@v2
105+
with:
106+
registry: ghcr.io
107+
username: ${{ github.actor }}
108+
password: ${{ secrets.GITHUB_TOKEN }}
109+
110+
- name: Verify image exists before deployment
111+
env:
112+
IMAGE_NAME: ${{ env.IMAGE_NAME }}
113+
IMAGE_TAG: ${{ github.sha }}
114+
run: .github/scripts/verify-image.sh
115+
116+
- name: Deploy to App Engine
117+
env:
118+
IMAGE_NAME: ${{ env.IMAGE_NAME }}
119+
IMAGE_TAG: ${{ github.sha }}
120+
SERVICE_ACCOUNT: github-deployment@policyengine-household-api.iam.gserviceaccount.com
121+
APP_YAML_PATH: ./gcp/policyengine_household_api/app.yaml
122+
AUTH0_ADDRESS_NO_DOMAIN: ${{ secrets.AUTH0_ADDRESS_NO_DOMAIN }}
123+
AUTH0_AUDIENCE_NO_DOMAIN: ${{ secrets.AUTH0_AUDIENCE_NO_DOMAIN }}
124+
USER_ANALYTICS_DB_USERNAME: ${{ secrets.USER_ANALYTICS_DB_USERNAME }}
125+
USER_ANALYTICS_DB_PASSWORD: ${{ secrets.USER_ANALYTICS_DB_PASSWORD }}
126+
USER_ANALYTICS_DB_CONNECTION_NAME: ${{ secrets.USER_ANALYTICS_DB_CONNECTION_NAME }}
127+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
128+
run: .github/scripts/deploy-app-engine.sh
129+
130+
- name: Set traffic to new version
131+
env:
132+
SERVICE_NAME: ${{ env.SERVICE_NAME }}
133+
VERSION: ${{ github.sha }}
134+
run: .github/scripts/set-traffic.sh
135+
136+
- name: Verify deployment
137+
env:
138+
SERVICE_NAME: ${{ env.SERVICE_NAME }}
139+
run: .github/scripts/verify-deployment.sh

‎.github/workflows/docker.yml‎

Lines changed: 0 additions & 38 deletions
This file was deleted.

0 commit comments

Comments
 (0)