Skip to content

Commit 0ec150d

Browse files
anth-volkanth-volk
authored
Merge pull request #3497 from PolicyEngine/fix/sim-gateway-auth-client-2
Add simulation-gateway auth client to API v1
2 parents 63da827 + 01d5bc1 commit 0ec150d

34 files changed

Lines changed: 1351 additions & 264 deletions

‎.github/request-simulation-model-versions.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ set -e
1313
#
1414
# Usage: ./request-simulation-model-versions.sh -us <us_version>
1515

16-
GATEWAY_URL="https://policyengine--policyengine-simulation-gateway-web-app.modal.run"
16+
GATEWAY_URL="${SIMULATION_API_URL:-https://policyengine--policyengine-simulation-gateway-web-app.modal.run}"
1717

1818
usage() {
1919
echo "Usage: $0 -us <us_version>"

‎.github/scripts/deploy_app_engine_version.sh‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
: "${APP_ENGINE_VERSION:?APP_ENGINE_VERSION is required}"
6+
7+
APP_ENGINE_PROMOTE="${APP_ENGINE_PROMOTE:-0}"
8+
APP_ENGINE_SERVICE_ACCOUNT="${APP_ENGINE_SERVICE_ACCOUNT:-github-deployment@policyengine-api.iam.gserviceaccount.com}"
9+
10+
cleanup() {
11+
rm -f app.yaml Dockerfile start.sh .gac.json .dbpw
12+
}
13+
14+
trap cleanup EXIT
15+
16+
python gcp/export.py
17+
cp gcp/policyengine_api/app.yaml .
18+
cp gcp/policyengine_api/Dockerfile .
19+
cp gcp/policyengine_api/start.sh .
20+
21+
gcloud config set app/cloud_build_timeout 2400
22+
23+
deploy_args=(
24+
app.yaml
25+
"--service-account=${APP_ENGINE_SERVICE_ACCOUNT}"
26+
"--version=${APP_ENGINE_VERSION}"
27+
)
28+
29+
if [[ -n "${APP_ENGINE_PROJECT:-}" ]]; then
30+
deploy_args+=("--project=${APP_ENGINE_PROJECT}")
31+
fi
32+
33+
if [[ "${APP_ENGINE_PROMOTE}" != "1" ]]; then
34+
deploy_args+=("--no-promote")
35+
fi
36+
37+
yes | gcloud app deploy "${deploy_args[@]}"

‎.github/scripts/get_app_engine_version_url.sh‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
: "${APP_ENGINE_VERSION:?APP_ENGINE_VERSION is required}"
6+
7+
APP_ENGINE_SERVICE="${APP_ENGINE_SERVICE:-default}"
8+
9+
browse_args=(
10+
"${APP_ENGINE_VERSION}"
11+
"--service=${APP_ENGINE_SERVICE}"
12+
"--no-launch-browser"
13+
)
14+
15+
if [[ -n "${APP_ENGINE_PROJECT:-}" ]]; then
16+
browse_args+=("--project=${APP_ENGINE_PROJECT}")
17+
fi
18+
19+
output="$(gcloud app versions browse "${browse_args[@]}" 2>&1)"
20+
url="$(printf '%s\n' "${output}" | grep -Eo 'https://[^[:space:]]+' | tail -n1)"
21+
22+
if [[ -z "${url}" ]]; then
23+
echo "Failed to determine App Engine version URL" >&2
24+
printf '%s\n' "${output}" >&2
25+
exit 1
26+
fi
27+
28+
printf '%s\n' "${url}"

‎.github/scripts/health_check.sh‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
health_url="${1:?health check URL is required}"
6+
timeout_seconds="${HEALTH_CHECK_TIMEOUT_SECONDS:-900}"
7+
interval_seconds="${HEALTH_CHECK_INTERVAL_SECONDS:-10}"
8+
deadline=$((SECONDS + timeout_seconds))
9+
10+
while (( SECONDS < deadline )); do
11+
if curl --silent --show-error --fail "${health_url}" >/dev/null; then
12+
exit 0
13+
fi
14+
sleep "${interval_seconds}"
15+
done
16+
17+
echo "Timed out waiting for healthy response from ${health_url}" >&2
18+
exit 1

‎.github/scripts/promote_app_engine_version.sh‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
: "${APP_ENGINE_VERSION:?APP_ENGINE_VERSION is required}"
6+
7+
APP_ENGINE_SERVICE="${APP_ENGINE_SERVICE:-default}"
8+
9+
promote_args=(
10+
"${APP_ENGINE_SERVICE}"
11+
"--splits=${APP_ENGINE_VERSION}=1"
12+
)
13+
14+
if [[ -n "${APP_ENGINE_PROJECT:-}" ]]; then
15+
promote_args+=("--project=${APP_ENGINE_PROJECT}")
16+
fi
17+
18+
gcloud app services set-traffic "${promote_args[@]}"

‎.github/scripts/validate_app_engine_deploy_env.sh‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
required=(
6+
SIMULATION_API_URL
7+
GATEWAY_AUTH_ISSUER
8+
GATEWAY_AUTH_AUDIENCE
9+
GATEWAY_AUTH_CLIENT_ID
10+
GATEWAY_AUTH_CLIENT_SECRET_RESOURCE
11+
)
12+
13+
missing=()
14+
15+
for name in "${required[@]}"; do
16+
if [[ -z "${!name:-}" ]]; then
17+
missing+=("$name")
18+
fi
19+
done
20+
21+
if [[ "${#missing[@]}" -gt 0 ]]; then
22+
echo "Missing required App Engine deployment configuration: ${missing[*]}" >&2
23+
exit 1
24+
fi

‎.github/workflows/push.yml‎

Lines changed: 178 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,14 @@ jobs:
2626
run: pip install ruff>=0.9.0
2727
- name: Format check with ruff
2828
run: ruff format --check .
29-
ensure-model-version-aligns-with-sim-api:
30-
name: Ensure model version aligns with simulation API
29+
30+
ensure-staging-model-version-aligns-with-sim-api:
31+
name: Ensure staging model version aligns with simulation API
3132
runs-on: ubuntu-latest
3233
if: |
3334
(github.repository == 'PolicyEngine/policyengine-api')
3435
&& (github.event.head_commit.message == 'Update PolicyEngine API')
36+
environment: staging
3537
steps:
3638
- name: Checkout repo
3739
uses: actions/checkout@v4
@@ -47,6 +49,9 @@ jobs:
4749
run: python3 .github/find-api-model-versions.py
4850
- name: Ensure full API and simulation API model versions are in sync
4951
run: ".github/request-simulation-model-versions.sh -us ${{ env.US_VERSION }} -uk ${{ env.UK_VERSION }}"
52+
env:
53+
SIMULATION_API_URL: ${{ secrets.SIMULATION_API_URL }}
54+
5055
versioning:
5156
name: Update versioning
5257
if: |
@@ -82,10 +87,11 @@ jobs:
8287
committer_name: Github Actions[bot]
8388
author_name: Github Actions[bot]
8489
message: Update PolicyEngine API
85-
deploy:
86-
name: Deploy API
90+
91+
publish-git-tag:
92+
name: Publish Git Tag
8793
runs-on: ubuntu-latest
88-
needs: ensure-model-version-aligns-with-sim-api
94+
needs: ensure-staging-model-version-aligns-with-sim-api
8995
if: |
9096
(github.repository == 'PolicyEngine/policyengine-api')
9197
&& (github.event.head_commit.message == 'Update PolicyEngine API')
@@ -98,25 +104,189 @@ jobs:
98104
python-version: "3.12"
99105
- name: Publish Git Tag
100106
run: ".github/publish-git-tag.sh"
107+
108+
deploy-staging:
109+
name: Deploy staging App Engine version
110+
runs-on: ubuntu-latest
111+
needs:
112+
- ensure-staging-model-version-aligns-with-sim-api
113+
- publish-git-tag
114+
if: |
115+
(github.repository == 'PolicyEngine/policyengine-api')
116+
&& (github.event.head_commit.message == 'Update PolicyEngine API')
117+
environment: staging
118+
outputs:
119+
version: ${{ steps.version.outputs.version }}
120+
url: ${{ steps.version_url.outputs.url }}
121+
steps:
122+
- name: Checkout repo
123+
uses: actions/checkout@v4
124+
- name: Setup Python
125+
uses: actions/setup-python@v5
126+
with:
127+
python-version: "3.12"
128+
- name: Compute staging version name
129+
id: version
130+
run: |
131+
echo "version=staging-${GITHUB_RUN_NUMBER}-${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
132+
- name: GCP authentication
133+
uses: "google-github-actions/auth@v2"
134+
with:
135+
credentials_json: "${{ secrets.GCP_SA_KEY }}"
136+
- name: Set up GCloud
137+
uses: "google-github-actions/setup-gcloud@v2"
138+
- name: Validate App Engine deployment configuration
139+
run: bash .github/scripts/validate_app_engine_deploy_env.sh
140+
env:
141+
SIMULATION_API_URL: ${{ secrets.SIMULATION_API_URL }}
142+
GATEWAY_AUTH_ISSUER: ${{ secrets.GATEWAY_AUTH_ISSUER }}
143+
GATEWAY_AUTH_AUDIENCE: ${{ secrets.GATEWAY_AUTH_AUDIENCE }}
144+
GATEWAY_AUTH_CLIENT_ID: ${{ secrets.GATEWAY_AUTH_CLIENT_ID }}
145+
GATEWAY_AUTH_CLIENT_SECRET_RESOURCE: ${{ secrets.GATEWAY_AUTH_CLIENT_SECRET_RESOURCE }}
146+
- name: Deploy staging version
147+
run: bash .github/scripts/deploy_app_engine_version.sh
148+
env:
149+
APP_ENGINE_VERSION: ${{ steps.version.outputs.version }}
150+
APP_ENGINE_PROMOTE: "0"
151+
POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }}
152+
GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GCP_SA_KEY }}
153+
POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }}
154+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
155+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
156+
HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
157+
SIMULATION_API_URL: ${{ secrets.SIMULATION_API_URL }}
158+
GATEWAY_AUTH_ISSUER: ${{ secrets.GATEWAY_AUTH_ISSUER }}
159+
GATEWAY_AUTH_AUDIENCE: ${{ secrets.GATEWAY_AUTH_AUDIENCE }}
160+
GATEWAY_AUTH_CLIENT_ID: ${{ secrets.GATEWAY_AUTH_CLIENT_ID }}
161+
GATEWAY_AUTH_CLIENT_SECRET_RESOURCE: ${{ secrets.GATEWAY_AUTH_CLIENT_SECRET_RESOURCE }}
162+
- name: Resolve staging version URL
163+
id: version_url
164+
run: |
165+
url="$(bash .github/scripts/get_app_engine_version_url.sh)"
166+
echo "url=${url}" >> "$GITHUB_OUTPUT"
167+
env:
168+
APP_ENGINE_VERSION: ${{ steps.version.outputs.version }}
169+
- name: Wait for staging version health
170+
run: bash .github/scripts/health_check.sh "${{ steps.version_url.outputs.url }}/readiness-check"
171+
172+
integration-tests-staging:
173+
name: Run staging integration tests
174+
runs-on: ubuntu-latest
175+
needs: deploy-staging
176+
if: |
177+
(github.repository == 'PolicyEngine/policyengine-api')
178+
&& (github.event.head_commit.message == 'Update PolicyEngine API')
179+
steps:
180+
- name: Checkout repo
181+
uses: actions/checkout@v4
182+
- name: Setup Python
183+
uses: actions/setup-python@v5
184+
with:
185+
python-version: "3.12"
186+
- name: Install staging test dependencies
187+
run: pip install pytest httpx
188+
- name: Run staging smoke test
189+
run: python -m pytest tests/integration/test_live_calculate.py tests/integration/test_live_economy.py -v
190+
env:
191+
API_BASE_URL: ${{ needs.deploy-staging.outputs.url }}
192+
STAGING_API_TEST_PROBE_ID: ${{ needs.deploy-staging.outputs.version }}
193+
194+
ensure-production-model-version-aligns-with-sim-api:
195+
name: Ensure production model version aligns with simulation API
196+
runs-on: ubuntu-latest
197+
needs: integration-tests-staging
198+
if: |
199+
(github.repository == 'PolicyEngine/policyengine-api')
200+
&& (github.event.head_commit.message == 'Update PolicyEngine API')
201+
environment: production
202+
steps:
203+
- name: Checkout repo
204+
uses: actions/checkout@v4
205+
- name: Setup Python
206+
uses: actions/setup-python@v5
207+
with:
208+
python-version: "3.12"
209+
- name: Install dependencies (required for finding API model versions)
210+
run: make install
211+
- name: Install jq (required only for GitHub Actions)
212+
run: sudo apt-get install -y jq
213+
- name: Find API model versions and write to environment variable
214+
run: python3 .github/find-api-model-versions.py
215+
- name: Ensure full API and simulation API model versions are in sync
216+
run: ".github/request-simulation-model-versions.sh -us ${{ env.US_VERSION }} -uk ${{ env.UK_VERSION }}"
217+
env:
218+
SIMULATION_API_URL: ${{ secrets.SIMULATION_API_URL }}
219+
220+
deploy-production:
221+
name: Deploy production App Engine version
222+
runs-on: ubuntu-latest
223+
needs: ensure-production-model-version-aligns-with-sim-api
224+
if: |
225+
(github.repository == 'PolicyEngine/policyengine-api')
226+
&& (github.event.head_commit.message == 'Update PolicyEngine API')
227+
environment: production
228+
steps:
229+
- name: Checkout repo
230+
uses: actions/checkout@v4
231+
- name: Setup Python
232+
uses: actions/setup-python@v5
233+
with:
234+
python-version: "3.12"
235+
- name: Compute production version name
236+
id: version
237+
run: |
238+
echo "version=prod-${GITHUB_RUN_NUMBER}-${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
101239
- name: GCP authentication
102240
uses: "google-github-actions/auth@v2"
103241
with:
104242
credentials_json: "${{ secrets.GCP_SA_KEY }}"
105243
- name: Set up GCloud
106244
uses: "google-github-actions/setup-gcloud@v2"
107-
- name: Deploy
108-
run: make deploy
245+
- name: Validate App Engine deployment configuration
246+
run: bash .github/scripts/validate_app_engine_deploy_env.sh
109247
env:
248+
SIMULATION_API_URL: ${{ secrets.SIMULATION_API_URL }}
249+
GATEWAY_AUTH_ISSUER: ${{ secrets.GATEWAY_AUTH_ISSUER }}
250+
GATEWAY_AUTH_AUDIENCE: ${{ secrets.GATEWAY_AUTH_AUDIENCE }}
251+
GATEWAY_AUTH_CLIENT_ID: ${{ secrets.GATEWAY_AUTH_CLIENT_ID }}
252+
GATEWAY_AUTH_CLIENT_SECRET_RESOURCE: ${{ secrets.GATEWAY_AUTH_CLIENT_SECRET_RESOURCE }}
253+
- name: Deploy production version
254+
run: bash .github/scripts/deploy_app_engine_version.sh
255+
env:
256+
APP_ENGINE_VERSION: ${{ steps.version.outputs.version }}
257+
APP_ENGINE_PROMOTE: "0"
110258
POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }}
111259
GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GCP_SA_KEY }}
112260
POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }}
113261
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
114262
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
115263
HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
264+
SIMULATION_API_URL: ${{ secrets.SIMULATION_API_URL }}
265+
GATEWAY_AUTH_ISSUER: ${{ secrets.GATEWAY_AUTH_ISSUER }}
266+
GATEWAY_AUTH_AUDIENCE: ${{ secrets.GATEWAY_AUTH_AUDIENCE }}
267+
GATEWAY_AUTH_CLIENT_ID: ${{ secrets.GATEWAY_AUTH_CLIENT_ID }}
268+
GATEWAY_AUTH_CLIENT_SECRET_RESOURCE: ${{ secrets.GATEWAY_AUTH_CLIENT_SECRET_RESOURCE }}
269+
- name: Resolve production version URL
270+
id: version_url
271+
run: |
272+
url="$(bash .github/scripts/get_app_engine_version_url.sh)"
273+
echo "url=${url}" >> "$GITHUB_OUTPUT"
274+
env:
275+
APP_ENGINE_VERSION: ${{ steps.version.outputs.version }}
276+
- name: Wait for production version health
277+
run: bash .github/scripts/health_check.sh "${{ steps.version_url.outputs.url }}/readiness-check"
278+
- name: Promote production version
279+
run: bash .github/scripts/promote_app_engine_version.sh
280+
env:
281+
APP_ENGINE_VERSION: ${{ steps.version.outputs.version }}
282+
116283
docker:
117284
name: Docker
118285
runs-on: ubuntu-latest
119-
needs: ensure-model-version-aligns-with-sim-api
286+
needs: deploy-production
287+
if: |
288+
(github.repository == 'PolicyEngine/policyengine-api')
289+
&& (github.event.head_commit.message == 'Update PolicyEngine API')
120290
permissions:
121291
contents: read
122292
packages: write

‎changelog.d/3496.added.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Add Auth0 client-credentials bearer auth for outbound simulation-gateway calls in API v1, including `GATEWAY_AUTH_REQUIRED`, startup validation for partial auth configuration, Google Secret Manager support for the gateway client secret via `GATEWAY_AUTH_CLIENT_SECRET_RESOURCE`, and staged App Engine version deployments with a staging smoke test before production promotion.

0 commit comments

Comments
 (0)