Skip to content

Commit ce2c1db

Browse files
anth-volkanth-volk
authored
Merge pull request #3649 from PolicyEngine/migration-pr3-cloud-run-candidate
Stage 3: deploy FastAPI shell as Cloud Run candidate
2 parents 9783df7 + cf629b5 commit ce2c1db

30 files changed

Lines changed: 1702 additions & 93 deletions

‎.github/scripts/build_cloud_run_image.sh‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
source .github/scripts/cloud_run_env.sh
6+
cloud_run_set_defaults
7+
8+
if [[ "${CLOUD_RUN_DRY_RUN:-0}" == "1" ]]; then
9+
cloud_run_run gcloud services enable run.googleapis.com artifactregistry.googleapis.com cloudbuild.googleapis.com --project "${CLOUD_RUN_PROJECT}"
10+
cloud_run_run gcloud artifacts repositories describe "${CLOUD_RUN_ARTIFACT_REPOSITORY}" --project "${CLOUD_RUN_PROJECT}" --location "${CLOUD_RUN_REGION}"
11+
cloud_run_run gcloud artifacts repositories create "${CLOUD_RUN_ARTIFACT_REPOSITORY}" --repository-format docker --location "${CLOUD_RUN_REGION}" --description "Docker repository for PolicyEngine API Cloud Run"
12+
cloud_run_run gcloud auth configure-docker "${CLOUD_RUN_REGION}-docker.pkg.dev" --quiet
13+
cloud_run_run docker build -f gcp/cloud_run/Dockerfile -t "${CLOUD_RUN_IMAGE_URI}" .
14+
cloud_run_run docker push "${CLOUD_RUN_IMAGE_URI}"
15+
exit 0
16+
fi
17+
18+
gcloud services enable run.googleapis.com artifactregistry.googleapis.com cloudbuild.googleapis.com --project "${CLOUD_RUN_PROJECT}"
19+
20+
if ! gcloud artifacts repositories describe "${CLOUD_RUN_ARTIFACT_REPOSITORY}" \
21+
--project "${CLOUD_RUN_PROJECT}" \
22+
--location "${CLOUD_RUN_REGION}" >/dev/null 2>&1; then
23+
gcloud artifacts repositories create "${CLOUD_RUN_ARTIFACT_REPOSITORY}" \
24+
--repository-format docker \
25+
--location "${CLOUD_RUN_REGION}" \
26+
--description "Docker repository for PolicyEngine API Cloud Run"
27+
fi
28+
29+
gcloud auth configure-docker "${CLOUD_RUN_REGION}-docker.pkg.dev" --quiet
30+
docker build -f gcp/cloud_run/Dockerfile -t "${CLOUD_RUN_IMAGE_URI}" .
31+
docker push "${CLOUD_RUN_IMAGE_URI}"

‎.github/scripts/check_changelog_fragment.sh‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
fragments="$(find changelog.d -type f ! -name '.gitkeep' | wc -l)"
6+
if [[ "${fragments}" -eq 0 ]]; then
7+
echo "::error::No changelog fragment found in changelog.d/"
8+
echo "Add one with: echo 'Description.' > changelog.d/\$(git branch --show-current).<type>.md"
9+
echo "Types: added, changed, fixed, removed, breaking"
10+
exit 1
11+
fi

‎.github/scripts/cloud_run_env.sh‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
#!/usr/bin/env bash
2+
3+
cloud_run_set_defaults() {
4+
CLOUD_RUN_PROJECT="${CLOUD_RUN_PROJECT:-policyengine-api}"
5+
CLOUD_RUN_REGION="${CLOUD_RUN_REGION:-us-central1}"
6+
CLOUD_RUN_SERVICE="${CLOUD_RUN_SERVICE:-policyengine-api}"
7+
CLOUD_RUN_ARTIFACT_REPOSITORY="${CLOUD_RUN_ARTIFACT_REPOSITORY:-policyengine-api}"
8+
CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT="${CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT:-policyengine-api-cr-runtime@policyengine-api.iam.gserviceaccount.com}"
9+
CLOUD_RUN_CLOUD_SQL_INSTANCE="${CLOUD_RUN_CLOUD_SQL_INSTANCE:-policyengine-api:us-central1:policyengine-api-data}"
10+
CLOUD_RUN_CPU="${CLOUD_RUN_CPU:-4}"
11+
CLOUD_RUN_MEMORY="${CLOUD_RUN_MEMORY:-16Gi}"
12+
CLOUD_RUN_TIMEOUT="${CLOUD_RUN_TIMEOUT:-300}"
13+
CLOUD_RUN_MIN_INSTANCES="${CLOUD_RUN_MIN_INSTANCES:-0}"
14+
CLOUD_RUN_MAX_INSTANCES="${CLOUD_RUN_MAX_INSTANCES:-1}"
15+
CLOUD_RUN_PORT="${CLOUD_RUN_PORT:-8080}"
16+
CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET="${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET:-policyengine-api-prod-db-password:latest}"
17+
CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET="${CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET:-policyengine-api-prod-github-microdata-token:latest}"
18+
CLOUD_RUN_ANTHROPIC_API_KEY_SECRET="${CLOUD_RUN_ANTHROPIC_API_KEY_SECRET:-policyengine-api-prod-anthropic-api-key:latest}"
19+
CLOUD_RUN_OPENAI_API_KEY_SECRET="${CLOUD_RUN_OPENAI_API_KEY_SECRET:-policyengine-api-prod-openai-api-key:latest}"
20+
CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET="${CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET:-policyengine-api-prod-hugging-face-token:latest}"
21+
22+
local sha
23+
sha="${GITHUB_SHA:-local}"
24+
CLOUD_RUN_IMAGE_TAG="${CLOUD_RUN_IMAGE_TAG:-${sha}}"
25+
CLOUD_RUN_IMAGE_URI="${CLOUD_RUN_IMAGE_URI:-${CLOUD_RUN_REGION}-docker.pkg.dev/${CLOUD_RUN_PROJECT}/${CLOUD_RUN_ARTIFACT_REPOSITORY}/${CLOUD_RUN_SERVICE}:${CLOUD_RUN_IMAGE_TAG}}"
26+
27+
local short_sha
28+
short_sha="${sha:0:7}"
29+
CLOUD_RUN_TAG="${CLOUD_RUN_TAG:-stage3-${GITHUB_RUN_NUMBER:-local}-${short_sha}}"
30+
31+
export CLOUD_RUN_PROJECT
32+
export CLOUD_RUN_REGION
33+
export CLOUD_RUN_SERVICE
34+
export CLOUD_RUN_ARTIFACT_REPOSITORY
35+
export CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT
36+
export CLOUD_RUN_CLOUD_SQL_INSTANCE
37+
export CLOUD_RUN_CPU
38+
export CLOUD_RUN_MEMORY
39+
export CLOUD_RUN_TIMEOUT
40+
export CLOUD_RUN_MIN_INSTANCES
41+
export CLOUD_RUN_MAX_INSTANCES
42+
export CLOUD_RUN_PORT
43+
export CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET
44+
export CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET
45+
export CLOUD_RUN_ANTHROPIC_API_KEY_SECRET
46+
export CLOUD_RUN_OPENAI_API_KEY_SECRET
47+
export CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET
48+
export CLOUD_RUN_IMAGE_TAG
49+
export CLOUD_RUN_IMAGE_URI
50+
export CLOUD_RUN_TAG
51+
}
52+
53+
cloud_run_require_env() {
54+
local missing=()
55+
local name
56+
57+
for name in "$@"; do
58+
if [[ -z "${!name:-}" ]]; then
59+
missing+=("${name}")
60+
fi
61+
done
62+
63+
if (( ${#missing[@]} > 0 )); then
64+
echo "Missing required Cloud Run deployment configuration: ${missing[*]}" >&2
65+
return 1
66+
fi
67+
}
68+
69+
cloud_run_run() {
70+
if [[ "${CLOUD_RUN_DRY_RUN:-0}" == "1" ]]; then
71+
printf '+'
72+
printf ' %q' "$@"
73+
printf '\n'
74+
return 0
75+
fi
76+
77+
"$@"
78+
}

‎.github/scripts/deploy_cloud_run_candidate.sh‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
source .github/scripts/cloud_run_env.sh
6+
cloud_run_set_defaults
7+
8+
bash .github/scripts/validate_cloud_run_deploy_env.sh
9+
10+
env_vars=(
11+
"POLICYENGINE_DB_INSTANCE_CONNECTION_NAME=${CLOUD_RUN_CLOUD_SQL_INSTANCE}"
12+
"POLICYENGINE_DB_USER=${POLICYENGINE_DB_USER:-policyengine}"
13+
"POLICYENGINE_DB_NAME=${POLICYENGINE_DB_NAME:-policyengine}"
14+
"SIMULATION_API_URL=${SIMULATION_API_URL}"
15+
"GATEWAY_AUTH_REQUIRED=1"
16+
"GATEWAY_AUTH_ISSUER=${GATEWAY_AUTH_ISSUER}"
17+
"GATEWAY_AUTH_AUDIENCE=${GATEWAY_AUTH_AUDIENCE}"
18+
"GATEWAY_AUTH_CLIENT_ID=${GATEWAY_AUTH_CLIENT_ID}"
19+
"GATEWAY_AUTH_CLIENT_SECRET_RESOURCE=${GATEWAY_AUTH_CLIENT_SECRET_RESOURCE}"
20+
"API_HOST_BACKEND=cloud_run"
21+
"SIM_FRONT_DOOR=old_gateway_direct"
22+
"SIM_COMPUTE_ECONOMY=old_gateway"
23+
"CLOUD_RUN_REVISION_TAG=${CLOUD_RUN_TAG}"
24+
)
25+
26+
secret_vars=(
27+
"POLICYENGINE_DB_PASSWORD=${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET}"
28+
"POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN=${CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET}"
29+
"ANTHROPIC_API_KEY=${CLOUD_RUN_ANTHROPIC_API_KEY_SECRET}"
30+
"OPENAI_API_KEY=${CLOUD_RUN_OPENAI_API_KEY_SECRET}"
31+
"HUGGING_FACE_TOKEN=${CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET}"
32+
)
33+
34+
set_env_vars="$(IFS='|'; echo "^|^${env_vars[*]}")"
35+
set_secret_vars="$(IFS='|'; echo "^|^${secret_vars[*]}")"
36+
37+
cloud_run_run gcloud run deploy "${CLOUD_RUN_SERVICE}" \
38+
--project "${CLOUD_RUN_PROJECT}" \
39+
--region "${CLOUD_RUN_REGION}" \
40+
--platform managed \
41+
--image "${CLOUD_RUN_IMAGE_URI}" \
42+
--tag "${CLOUD_RUN_TAG}" \
43+
--no-traffic \
44+
--allow-unauthenticated \
45+
--execution-environment gen2 \
46+
--service-account "${CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT}" \
47+
--add-cloudsql-instances "${CLOUD_RUN_CLOUD_SQL_INSTANCE}" \
48+
--port "${CLOUD_RUN_PORT}" \
49+
--cpu "${CLOUD_RUN_CPU}" \
50+
--memory "${CLOUD_RUN_MEMORY}" \
51+
--timeout "${CLOUD_RUN_TIMEOUT}" \
52+
--min-instances "${CLOUD_RUN_MIN_INSTANCES}" \
53+
--max-instances "${CLOUD_RUN_MAX_INSTANCES}" \
54+
--set-env-vars "${set_env_vars}" \
55+
--set-secrets "${set_secret_vars}"

‎.github/scripts/get_cloud_run_service_url.sh‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
source .github/scripts/cloud_run_env.sh
6+
cloud_run_set_defaults
7+
8+
if [[ "${CLOUD_RUN_DRY_RUN:-0}" == "1" ]]; then
9+
echo "https://${CLOUD_RUN_SERVICE}-dry-run.a.run.app"
10+
exit 0
11+
fi
12+
13+
gcloud run services describe "${CLOUD_RUN_SERVICE}" \
14+
--project "${CLOUD_RUN_PROJECT}" \
15+
--region "${CLOUD_RUN_REGION}" \
16+
--platform managed \
17+
--format 'value(status.url)'

‎.github/scripts/get_cloud_run_tag_url.sh‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
source .github/scripts/cloud_run_env.sh
6+
cloud_run_set_defaults
7+
8+
if [[ "${CLOUD_RUN_DRY_RUN:-0}" == "1" ]]; then
9+
echo "https://${CLOUD_RUN_TAG}---${CLOUD_RUN_SERVICE}-dry-run.a.run.app"
10+
exit 0
11+
fi
12+
13+
gcloud run services describe "${CLOUD_RUN_SERVICE}" \
14+
--project "${CLOUD_RUN_PROJECT}" \
15+
--region "${CLOUD_RUN_REGION}" \
16+
--platform managed \
17+
--format json | python -c '
18+
import json
19+
import os
20+
import sys
21+
22+
service = json.load(sys.stdin)
23+
tag = os.environ["CLOUD_RUN_TAG"]
24+
for traffic_target in service.get("status", {}).get("traffic", []):
25+
if traffic_target.get("tag") == tag and traffic_target.get("url"):
26+
print(traffic_target["url"])
27+
raise SystemExit(0)
28+
29+
print(f"Failed to determine Cloud Run URL for tag {tag}", file=sys.stderr)
30+
raise SystemExit(1)
31+
'

‎.github/scripts/promote_cloud_run_tag.sh‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
source .github/scripts/cloud_run_env.sh
6+
cloud_run_set_defaults
7+
8+
cloud_run_require_env \
9+
CLOUD_RUN_PROJECT \
10+
CLOUD_RUN_REGION \
11+
CLOUD_RUN_SERVICE \
12+
CLOUD_RUN_TAG
13+
14+
cloud_run_run gcloud run services update-traffic "${CLOUD_RUN_SERVICE}" \
15+
--project "${CLOUD_RUN_PROJECT}" \
16+
--region "${CLOUD_RUN_REGION}" \
17+
--platform managed \
18+
--to-tags "${CLOUD_RUN_TAG}=100"

‎.github/scripts/sync_cloud_run_secrets.sh‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
set +x
5+
6+
CLOUD_RUN_PROJECT="${CLOUD_RUN_PROJECT:-policyengine-api}"
7+
8+
require_env() {
9+
local env_name="$1"
10+
if [[ -z "${!env_name:-}" ]]; then
11+
echo "::error::Missing required workflow environment ${env_name}."
12+
exit 1
13+
fi
14+
}
15+
16+
sync_secret() {
17+
local env_name="$1"
18+
local secret_name="$2"
19+
local secret_value="${!env_name:-}"
20+
21+
if [[ -z "${secret_value}" ]]; then
22+
echo "::error::Missing required GitHub secret ${env_name}."
23+
exit 1
24+
fi
25+
26+
if ! gcloud secrets describe "${secret_name}" \
27+
--project "${CLOUD_RUN_PROJECT}" >/dev/null 2>&1; then
28+
gcloud secrets create "${secret_name}" \
29+
--project "${CLOUD_RUN_PROJECT}" \
30+
--replication-policy automatic
31+
fi
32+
33+
printf '%s' "${secret_value}" | gcloud secrets versions add \
34+
"${secret_name}" \
35+
--project "${CLOUD_RUN_PROJECT}" \
36+
--data-file=- >/dev/null
37+
38+
gcloud secrets add-iam-policy-binding "${secret_name}" \
39+
--project "${CLOUD_RUN_PROJECT}" \
40+
--member "serviceAccount:${CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT}" \
41+
--role roles/secretmanager.secretAccessor >/dev/null
42+
43+
echo "Synced ${env_name} to Secret Manager secret ${secret_name}."
44+
}
45+
46+
require_env CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT
47+
48+
sync_secret POLICYENGINE_DB_PASSWORD policyengine-api-prod-db-password
49+
sync_secret POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN policyengine-api-prod-github-microdata-token
50+
sync_secret ANTHROPIC_API_KEY policyengine-api-prod-anthropic-api-key
51+
sync_secret OPENAI_API_KEY policyengine-api-prod-openai-api-key
52+
sync_secret HUGGING_FACE_TOKEN policyengine-api-prod-hugging-face-token

‎.github/scripts/validate_cloud_run_deploy_env.sh‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
source .github/scripts/cloud_run_env.sh
6+
cloud_run_set_defaults
7+
8+
cloud_run_require_env \
9+
CLOUD_RUN_PROJECT \
10+
CLOUD_RUN_REGION \
11+
CLOUD_RUN_SERVICE \
12+
CLOUD_RUN_ARTIFACT_REPOSITORY \
13+
CLOUD_RUN_IMAGE_URI \
14+
CLOUD_RUN_TAG \
15+
CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT \
16+
CLOUD_RUN_CLOUD_SQL_INSTANCE \
17+
CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET \
18+
CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET \
19+
CLOUD_RUN_ANTHROPIC_API_KEY_SECRET \
20+
CLOUD_RUN_OPENAI_API_KEY_SECRET \
21+
CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET \
22+
SIMULATION_API_URL \
23+
GATEWAY_AUTH_ISSUER \
24+
GATEWAY_AUTH_AUDIENCE \
25+
GATEWAY_AUTH_CLIENT_ID \
26+
GATEWAY_AUTH_CLIENT_SECRET_RESOURCE

‎.github/workflows/pr.yml‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -36,14 +36,7 @@ jobs:
3636
steps:
3737
- uses: actions/checkout@v4
3838
- name: Check for changelog fragment
39-
run: |
40-
FRAGMENTS=$(find changelog.d -type f ! -name '.gitkeep' | wc -l)
41-
if [ "$FRAGMENTS" -eq 0 ]; then
42-
echo "::error::No changelog fragment found in changelog.d/"
43-
echo "Add one with: echo 'Description.' > changelog.d/\$(git branch --show-current).<type>.md"
44-
echo "Types: added, changed, fixed, removed, breaking"
45-
exit 1
46-
fi
39+
run: bash .github/scripts/check_changelog_fragment.sh
4740
test_container_builds:
4841
name: Docker
4942
runs-on: ubuntu-latest
@@ -61,6 +54,16 @@ jobs:
6154
password: ${{ secrets.GITHUB_TOKEN }}
6255
- name: Build container
6356
run: docker build -t ghcr.io/policyengine/policyengine docker
57+
test_cloud_run_container_builds:
58+
name: Cloud Run container
59+
runs-on: ubuntu-latest
60+
permissions:
61+
contents: read
62+
steps:
63+
- name: Checkout repo
64+
uses: actions/checkout@v4
65+
- name: Build Cloud Run container
66+
run: docker build -f gcp/cloud_run/Dockerfile -t policyengine-api-cloud-run:test .
6467
test_env_vars:
6568
name: Test environment variables
6669
runs-on: ubuntu-latest

0 commit comments

Comments
 (0)