feat: Update environment configuration and documentation for secret management and deployment

Author: ColinDaglish

.env.example

@@ -1,7 +1,14 @@
 # GitHub App settings
 GITHUB_APP_ID=
-GITHUB_PRIVATE_KEY=
-GITHUB_WEBHOOK_SECRET=
+
+# Secrets are provided as files.
+# Local dev (running the app on your machine): use the tutorial's local-secrets folder.
+GITHUB_PRIVATE_KEY_FILE=./local-secrets/github-private-key/private_key
+GITHUB_WEBHOOK_SECRET_FILE=./local-secrets/github-webhook-secret/webhook_secret
+
+# Docker/Cloud Run (secrets mounted into the container): these are the in-container paths.
+# GITHUB_PRIVATE_KEY_FILE=/var/secrets/github-private-key/private_key
+# GITHUB_WEBHOOK_SECRET_FILE=/var/secrets/github-webhook-secret/webhook_secret
 
 # Optional: limit accepted event types (comma-separated, e.g. pull_request,push)
 GITHUB_ACCEPTED_EVENTS=

.gitignore

@@ -195,10 +195,39 @@ celerybeat.pid
 
 # Environments
 .env
+.env.*
+!.env.example
 .venv
 env/
 venv/
 
+### Local secrets (do not commit) ###
+# Used by docs/tutorial/04-deploy-and-verify.md for local Secret Manager parity.
+local-secrets/
+
+# Common key/cert bundles that may contain private material.
+*.pem
+*.key
+*.p12
+*.pfx
+*.jks
+*.der
+
+### Terraform state & local working dirs (never commit) ###
+**/.terraform/
+**/.terraform.lock.hcl
+**/terraform.tfstate
+**/terraform.tfstate.*
+**/crash.log
+**/crash.*.log
+**/override.tf
+**/override.tf.json
+**/*_override.tf
+**/*_override.tf.json
+
+# Repo uses terraform.tfvars.example as the template; keep local tfvars untracked.
+**/terraform.tfvars
+
 # Terraform
 *.tfvars
 *.tfvars.json

README.md

@@ -5,6 +5,8 @@
 
 A GitHub App backend for secure webhook processing and automation, deployed on Google Cloud Run. The app validates GitHub webhook signatures, manages installation tokens, and routes events for further processing. Infrastructure is provisioned with Terraform, and security is enforced via pre-commit hooks and CI scans.
 
+If you want to replicate the full setup (GCP remote state, Terraform apply phases, GitHub App creation, deploy + verification), follow the end-to-end tutorial in `docs/tutorial/README.md`.
+
 ## Features
 
 - FastAPI-based webhook handler (`/webhooks/github`)
@@ -34,8 +36,8 @@ A GitHub App backend for secure webhook processing and automation, deployed on G
 
 3. Set required environment variables:
    - `GITHUB_APP_ID`
-   - `GITHUB_PRIVATE_KEY`
-   - `GITHUB_WEBHOOK_SECRET`
+   - `GITHUB_PRIVATE_KEY_FILE`
+   - `GITHUB_WEBHOOK_SECRET_FILE`
    - `GITHUB_ACCEPTED_EVENTS` (optional, comma-separated)
 
 ## Usage
@@ -51,16 +53,20 @@ A GitHub App backend for secure webhook processing and automation, deployed on G
 
 ## Deployment
 
-- Build and deploy with Docker and Google Cloud Build:
-  - See `cloudbuild.yaml` and `deploy.sh` for build and deployment steps.
-- Infrastructure setup:
-  - See `infra/terraform/README.md` for Terraform instructions.
+- Canonical workflow: Terraform-managed deployment (Cloud Run + API Gateway) with Artifact Registry images and secrets mounted from Secret Manager.
+   - Start here: `docs/tutorial/README.md`
+   - Terraform module docs: `infra/terraform/README.md`
+
+- Optional helpers:
+   - `deploy.sh` can build and push a container image to Artifact Registry.
+   - `cloudbuild.yaml` can build and push an image in Cloud Build.
+   These helpers intentionally do not provision infra; Terraform remains the source of truth.
 
 ## Security
 
 - Pre-commit hooks scan for secrets and large files.
 - CI workflow runs Checkov and Trivy scans on every push/PR.
-- Sensitive values are managed via environment variables and never committed.
+- Sensitive values are managed via Secret Manager (Cloud Run) or local secret files (development) and never committed.
 
 ## Project Workflow
 

cloudbuild.yaml

@@ -1,40 +1,11 @@
 steps:
   - name: gcr.io/cloud-builders/docker
-    args: ["build", "-t", "gcr.io/$PROJECT_ID/${_SERVICE_NAME}:$COMMIT_SHA", "."]
+    args: ["build", "-t", "${_REGION}-docker.pkg.dev/$PROJECT_ID/${_ARTIFACT_REPO}/${_SERVICE_NAME}:$COMMIT_SHA", "."]
   - name: gcr.io/cloud-builders/docker
-    args: ["push", "gcr.io/$PROJECT_ID/${_SERVICE_NAME}:$COMMIT_SHA"]
-  - name: gcr.io/google.com/cloudsdktool/cloud-sdk
-    entrypoint: bash
-    args:
-      - -eu
-      - -c
-      - |
-        if [[ -z "${_GITHUB_APP_ID}" ]]; then
-          echo "ERROR: Missing required substitution _GITHUB_APP_ID (your GitHub App ID)."
-          echo "Set it on your trigger or pass: gcloud builds submit --substitutions=_GITHUB_APP_ID=123456"
-          exit 1
-        fi
-  - name: gcr.io/google.com/cloudsdktool/cloud-sdk
-    entrypoint: gcloud
-    args:
-      - run
-      - deploy
-      - "${_SERVICE_NAME}"
-      - --image
-      - "gcr.io/$PROJECT_ID/${_SERVICE_NAME}:$COMMIT_SHA"
-      - --region
-      - "${_REGION}"
-      - --platform
-      - managed
-      - --set-env-vars
-      - "GITHUB_APP_ID=${_GITHUB_APP_ID},GITHUB_ACCEPTED_EVENTS=pull_request,GITHUB_PRIVATE_KEY_FILE=/var/secrets/github_private_key,GITHUB_WEBHOOK_SECRET_FILE=/var/secrets/github_webhook_secret"
-      - --secret
-      - github-app-private-key=/var/secrets/github_private_key
-      - --secret
-      - github-webhook-secret=/var/secrets/github_webhook_secret
+    args: ["push", "${_REGION}-docker.pkg.dev/$PROJECT_ID/${_ARTIFACT_REPO}/${_SERVICE_NAME}:$COMMIT_SHA"]
 substitutions:
   _SERVICE_NAME: ons-github-app
   _REGION: europe-west2
-  _GITHUB_APP_ID: ""
+  _ARTIFACT_REPO: ons-github-app
 images:
-  - "gcr.io/$PROJECT_ID/${_SERVICE_NAME}:$COMMIT_SHA"
+  - "${_REGION}-docker.pkg.dev/$PROJECT_ID/${_ARTIFACT_REPO}/${_SERVICE_NAME}:$COMMIT_SHA"

deploy.sh

@@ -2,20 +2,23 @@
 set -euo pipefail
 
 : "${PROJECT_ID:?Set PROJECT_ID}"
-: "${SERVICE_NAME:=ons-github-app}"
 : "${REGION:=europe-west2}"
-: "${GITHUB_APP_ID:?Set GITHUB_APP_ID}"
+: "${ARTIFACT_REPO:=ons-github-app}"
+: "${SERVICE_NAME:=ons-github-app}"
+: "${TAG:=latest}"
 
-: "${GITHUB_ACCEPTED_EVENTS:=pull_request}"
+# This repo's canonical deploy path is Terraform (Cloud Run + API Gateway).
+# This script is a convenience to build and push the container image to Artifact Registry.
+#
+# After running, set the resulting image URI as `image = "..."` in `infra/terraform/terraform.tfvars`
+# and run `terraform apply`.
 
-IMAGE="gcr.io/${PROJECT_ID}/${SERVICE_NAME}:latest"
+IMAGE="${REGION}-docker.pkg.dev/${PROJECT_ID}/${ARTIFACT_REPO}/${SERVICE_NAME}:${TAG}"
 
-gcloud builds submit --tag "${IMAGE}" .
+docker buildx build --platform linux/amd64 \
+  -t "${IMAGE}" \
+  --push .
 
-gcloud run deploy "${SERVICE_NAME}" \
-  --image "${IMAGE}" \
-  --region "${REGION}" \
-  --platform managed \
-  --set-env-vars "GITHUB_APP_ID=${GITHUB_APP_ID},GITHUB_ACCEPTED_EVENTS=${GITHUB_ACCEPTED_EVENTS},GITHUB_PRIVATE_KEY_FILE=/var/secrets/github_private_key,GITHUB_WEBHOOK_SECRET_FILE=/var/secrets/github_webhook_secret" \
-  --secret github-app-private-key=/var/secrets/github_private_key \
-  --secret github-webhook-secret=/var/secrets/github_webhook_secret
+echo
+echo "Pushed image: ${IMAGE}"
+echo "Next: update infra/terraform/terraform.tfvars (image = \"${IMAGE}\") and run terraform apply"

docs/tutorial/01-prereqs.md

@@ -36,13 +36,18 @@ Recommended:
 
 ### Secrets strategy (best practice)
 
-- **Local dev**: you can pass secrets via environment variables (e.g. `docker run --env-file .env ...`).
-- **Cloud Run**: secrets should not be plain env vars.
-  - Store secret values in **Secret Manager**.
-  - Mount them into the container as **files**.
-  - The app reads them via:
-    - `GITHUB_PRIVATE_KEY_FILE`
-    - `GITHUB_WEBHOOK_SECRET_FILE`
+This repo documents a single secrets strategy for both local development and Cloud Run:
+
+- Secrets are stored as **files**.
+- The app is configured with **non-secret** environment variables that point at those files:
+  - `GITHUB_PRIVATE_KEY_FILE`
+  - `GITHUB_WEBHOOK_SECRET_FILE`
+
+Local development:
+- Create a `./local-secrets/` folder (ignored by git) and point the `*_FILE` variables at it.
+
+Cloud Run:
+- Store secret values in **Secret Manager** and mount them into the container as **files**.
 
 This keeps secret values out of container images, build logs, and Terraform state.
 

docs/tutorial/04-deploy-and-verify.md

@@ -41,7 +41,6 @@ Edit `terraform.tfvars`:
 
 ```bash
 cd infra/terraform
-terraform init
 terraform apply
 ```
 
@@ -57,55 +56,80 @@ What you should get after phase 1:
 
 This is intentionally **outside Terraform** so secret values do not end up in Terraform state.
 
+This tutorial uses a single approach: secrets are first stored as local files under `./local-secrets/`, then:
+
+- those same files are used for local runs
+- and their contents are uploaded to Secret Manager (as secret versions)
+
+Create the local secret files now:
+
+```bash
+mkdir -p ./local-secrets/github-private-key ./local-secrets/github-webhook-secret
+cp /path/to/private-key.pem ./local-secrets/github-private-key/private_key
+printf "%s" "<your-webhook-secret>" > ./local-secrets/github-webhook-secret/webhook_secret
+```
+
 ### 2.1 Webhook secret
 
 ```bash
-echo -n "<your-webhook-secret>" | gcloud secrets versions add github-webhook-secret --data-file=-
+gcloud secrets versions add github-webhook-secret --data-file=./local-secrets/github-webhook-secret/webhook_secret
 ```
 
 ### 2.2 Private key
 
 ```bash
-gcloud secrets versions add github-private-key --data-file=/path/to/private-key.pem
+gcloud secrets versions add github-private-key --data-file=./local-secrets/github-private-key/private_key
 ```
 
 ## 3) Test locally (before deploying)
 
-### Option A: Local env vars (simple)
+This repo assumes a single local strategy: secrets are stored as files in `./local-secrets/` and the app reads them via `GITHUB_PRIVATE_KEY_FILE` and `GITHUB_WEBHOOK_SECRET_FILE`.
+
+You created `./local-secrets/` in step 2; the commands below read from those files.
+
+### 3.1 Run locally (no Docker)
 
 Create a `.env` (do not commit):
 
 ```env
 GITHUB_APP_ID=123456
-GITHUB_PRIVATE_KEY=<paste-private-key-as-a-single-line-with-literal-\n-escapes>
-GITHUB_WEBHOOK_SECRET=<your-webhook-secret>
+GITHUB_PRIVATE_KEY_FILE=./local-secrets/github-private-key/private_key
+GITHUB_WEBHOOK_SECRET_FILE=./local-secrets/github-webhook-secret/webhook_secret
 GITHUB_ACCEPTED_EVENTS=pull_request
 ```
 
-Run:
+Load it into your shell and run the API:
+
+```bash
+set -a
+source .env
+set +a
+
+uvicorn src.app:app --host 0.0.0.0 --port 8080
+```
+
+In another terminal:
 
 ```bash
-docker build -t ons-github-app:local .
-docker run --rm -p 8080:8080 --env-file .env ons-github-app:local
 curl http://localhost:8080/healthz
 ```
 
-### Option B: Local secret *files* (parity with Cloud Run)
+### 3.2 Run in Docker (parity with Cloud Run)
 
-This mirrors how Cloud Run runs the container.
+This mirrors how Cloud Run runs the container by mounting `./local-secrets` into the container at `/var/secrets`.
 
 ```bash
-mkdir -p ./local-secrets
-printf "%s" "<your-webhook-secret>" > ./local-secrets/github_webhook_secret
-cp /path/to/private-key.pem ./local-secrets/github_private_key
+docker build -t ons-github-app:local .
 
 docker run --rm -p 8080:8080 \
   -e GITHUB_APP_ID=123456 \
   -e GITHUB_ACCEPTED_EVENTS=pull_request \
-  -e GITHUB_PRIVATE_KEY_FILE=/var/secrets/github_private_key \
-  -e GITHUB_WEBHOOK_SECRET_FILE=/var/secrets/github_webhook_secret \
+  -e GITHUB_PRIVATE_KEY_FILE=/var/secrets/github-private-key/private_key \
+  -e GITHUB_WEBHOOK_SECRET_FILE=/var/secrets/github-webhook-secret/webhook_secret \
   -v "${PWD}/local-secrets:/var/secrets:ro" \
   ons-github-app:local
+
+curl http://localhost:8080/healthz
 ```
 
 ## 4) Build and push the image

infra/terraform/README.md

@@ -18,10 +18,17 @@ This folder provisions all required Google Cloud resources for the ons-github-ap
 
 2. Edit `terraform.tfvars` to set your project, region, and (optionally) the image URI.
 
-3. Initialize and apply the Terraform configuration:
+3. Initialize the Terraform configuration.
+
+This module uses a GCS backend which is intentionally configured at init time (bucket names are project-specific):
+
+```bash
+terraform init -backend-config="bucket=<state_bucket_name>"
+```
+
+4. Apply the Terraform configuration:
 
 ```bash
-   terraform init
    terraform apply
 ```
 
@@ -44,6 +51,8 @@ This folder provisions all required Google Cloud resources for the ons-github-ap
    1) First apply with `image = ""` provisions APIs, service account, Artifact Registry, and Secret Manager secrets.
    2) After pushing an image and adding secret versions, set `image` and apply again to create Cloud Run + API Gateway.
 
+For the end-to-end walkthrough (including remote state bootstrap and GitHub App setup), see `docs/tutorial/README.md`.
+
 ## Outputs
 
 After a successful apply, Terraform will output:

infra/terraform/main.tf

@@ -53,16 +53,21 @@ locals {
 
 # Enables required GCP APIs for Cloud Run, Artifact Registry, IAM, and API Gateway
 resource "google_project_service" "services" {
-  for_each           = toset(var.services)
-  project            = var.project_id
-  service            = each.value
-  disable_on_destroy = true
+  for_each = toset(var.services)
+  project  = var.project_id
+  service  = each.value
+  # Disabling APIs on destroy can break Terraform refresh/destroy of remaining
+  # resources (e.g. API Gateway returning 403 if apigateway.googleapis.com is disabled).
+  # Keep services enabled to make teardown reliable and repeatable.
+  disable_on_destroy = false
 }
 
 # Service account for the Cloud Run app
 resource "google_service_account" "app" {
   account_id   = "${var.service_name}-sa"
   display_name = "${var.service_name} service account"
+
+  depends_on = [google_project_service.services]
 }
 
 resource "google_kms_crypto_key_iam_member" "crypto_key" {
@@ -165,6 +170,8 @@ resource "google_api_gateway_api" "webhook" {
   provider = google-beta
   api_id   = "${var.service_name}-webhook"
   project  = var.project_id
+
+  depends_on = [google_project_service.services]
 }
 
 # API Gateway config using OpenAPI spec (api-config.yaml)
@@ -218,6 +225,8 @@ resource "google_secret_manager_secret" "github_private_key" {
       }
     }
   }
+
+  depends_on = [google_project_service.services]
 }
 
 resource "google_secret_manager_secret_iam_member" "github_private_key_accessor" {
@@ -238,6 +247,8 @@ resource "google_secret_manager_secret" "github_webhook_secret" {
       }
     }
   }
+
+  depends_on = [google_project_service.services]
 }
 
 resource "google_secret_manager_secret_iam_member" "github_webhook_secret_accessor" {