Merge pull request #71 from openplans/iac-setup

IAC Setup

Author: mjumbewu

.env.template

@@ -1,3 +1,7 @@
+# Server Configuration
+ALLOWED_HOSTS='*'
+SECRET_KEY='pbv(g=%7$$4rzvl88e24etn57-%n0uw-@y*=7ak422_3!zrc9+'
+
 # This is set up to use the PostGIS container spun up by docker-compose
 # in the root of this repository. If you are using a different database, you will
 # need to update the DATABASE_URL variable below.

.github/workflows/test.yml

@@ -0,0 +1,29 @@
+name: Test
+
+on:
+  push:
+    branches: [ master, main ]
+  pull_request:
+    branches: [ master, main ]
+
+jobs:
+  test:
+    runs-on: ubuntu-24.04  # Use 24.04 for Podman 4.x (matches local dev environment)
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: '3.12'
+
+    - name: Install Podman Compose
+      run: |
+        pip install podman-compose
+
+    - name: Build Images
+      run: make build
+
+    - name: Run Tests
+      run: make test

.gitignore

@@ -2,12 +2,13 @@
 db/*.sqlite3
 log/*.log
 
-# tmp
+# tmp & cache
 tmp/
 .sass-cache/
 profiling/
 attachments/
 build/
+.terraform/
 
 # cover_me generated
 coverage
@@ -26,6 +27,7 @@ coverage.data
 # environment
 env*/
 .env*
+.auto.tfvars
 
 # Local project settings
 src/project/local_settings.py

.travis.yml

@@ -3,25 +3,42 @@ FROM ubuntu:24.04
 # Install Python & GeoDjango dependencies
 RUN apt update && \
     apt install -y \
-        libpq-dev \
-        libproj-dev \
-        gdal-bin \
-        python3 \
-        python3-pip && \
+    libpq-dev \
+    libproj-dev \
+    gdal-bin \
+    python3 \
+    python3-pip \
+    python3-venv && \
     apt clean
 
+# Create a virtual environment
+ENV VIRTUAL_ENV=/opt/venv
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+ENV PYTHONUNBUFFERED=1
+
 # Install Python dependencies
 COPY requirements.txt /tmp/requirements.txt
-RUN pip3 install -r /tmp/requirements.txt --break-system-packages
+RUN pip install --no-cache-dir -r /tmp/requirements.txt
 
 # Copy the application code to the container
 COPY src /app
 WORKDIR /app
 
 # Run collectstatic to gather static files
-RUN REDIS_URL="redis://temp_value/" \
+# We pass dummy values for REDIS_URL and SECRET_KEY to ensure settings.py loads without error
+RUN REDIS_URL="redis://dummy:6379/0" \
+    SECRET_KEY="dummy" \
+    ALLOWED_HOSTS="*" \
     python3 manage.py collectstatic --noinput
 
-# Expose the port the app runs on
+# Copy gunicorn config
 COPY gunicorn.conf.py /app/gunicorn.conf.py
+
+# Expose the port the app runs on
 EXPOSE 8000
+
+# Default command
+CMD ["sh", "-c", "gunicorn project.wsgi --pythonpath src --workers 3 --config gunicorn.conf.py --bind 0.0.0.0:${PORT:-8000}"]
+
+

Containerfile

@@ -0,0 +1,39 @@
+.PHONY: test-env test test-clean build gcp-push gcp-restart gcp-deploy
+
+# Build the container image
+build:
+	podman build -t shareabouts-api -f Containerfile .
+
+# Push image to GCP Container Registry
+# Requires: PROJECT_ID, ENVIRONMENT_NAME environment variables
+gcp-push:
+	@if [ -z "$(PROJECT_ID)" ]; then echo "Error: PROJECT_ID is not set"; exit 1; fi
+	@if [ -z "$(ENVIRONMENT_NAME)" ]; then echo "Error: ENVIRONMENT_NAME is not set"; exit 1; fi
+	podman tag shareabouts-api gcr.io/$(PROJECT_ID)/shareabouts-api:latest-$(ENVIRONMENT_NAME)
+	podman push gcr.io/$(PROJECT_ID)/shareabouts-api:latest-$(ENVIRONMENT_NAME)
+
+# Restart the Cloud Run service with the latest image
+# Requires: PROJECT_ID, ENVIRONMENT_NAME, SERVICE_NAME, REGION environment variables
+gcp-restart:
+	@if [ -z "$(PROJECT_ID)" ]; then echo "Error: PROJECT_ID is not set"; exit 1; fi
+	@if [ -z "$(ENVIRONMENT_NAME)" ]; then echo "Error: ENVIRONMENT_NAME is not set"; exit 1; fi
+	@if [ -z "$(SERVICE_NAME)" ]; then echo "Error: SERVICE_NAME is not set"; exit 1; fi
+	@if [ -z "$(REGION)" ]; then echo "Error: REGION is not set"; exit 1; fi
+	gcloud run services update $(SERVICE_NAME)-$(ENVIRONMENT_NAME) \
+		--region $(REGION) \
+		--image gcr.io/$(PROJECT_ID)/shareabouts-api:latest-$(ENVIRONMENT_NAME)
+
+# Full deployment: build, push, and restart
+gcp-deploy: build gcp-push gcp-restart
+
+# Stub .env file
+test-env:
+	cp .env.template .env
+
+# Run tests in a clean container environment
+test: test-env test-clean
+	podman-compose run --rm test
+
+# Just clean up containers
+test-clean:
+	podman-compose down --remove-orphans 2>/dev/null || true

Makefile

@@ -1,17 +1,17 @@
-version: '3.8'
-
 services:
   init:
     build:
       context: .
       dockerfile: Containerfile
     command: >
       sh -c "
-        python3 manage.py migrate --noinput && 
+        python3 manage.py migrate --noinput &&
         python3 manage.py ensuresuperuser --noinput &&
         python3 manage.py createdefaultdataset
       "
     env_file: .env
+    environment:
+      - REDIS_URL=redis://redis:6379/0
     depends_on:
       db: {"condition": "service_healthy"}
       redis: {"condition": "service_healthy"}
@@ -21,8 +21,9 @@ services:
     build:
       context: .
       dockerfile: Containerfile
-    command: gunicorn project.wsgi --pythonpath src --workers ${WORKERS} --config gunicorn.conf.py --bind 0.0.0.0:8000
     env_file: .env
+    environment:
+      - REDIS_URL=redis://redis:6379/0
     ports:
       - "8000:8000"
     depends_on:
@@ -38,6 +39,20 @@ services:
     env_file: .env
     environment:
       C_FORCE_ROOT: "true"
+      REDIS_URL: "redis://redis:6379/0"
+    depends_on:
+      db: {"condition": "service_healthy"}
+      redis: {"condition": "service_healthy"}
+      init: {"condition": "service_completed_successfully"}
+
+  test:
+    build:
+      context: .
+      dockerfile: Containerfile
+    command: python3 manage.py test .
+    env_file: .env
+    environment:
+      - REDIS_URL=redis://redis:6379/0
     depends_on:
       db: {"condition": "service_healthy"}
       redis: {"condition": "service_healthy"}

compose.yml

@@ -80,3 +80,93 @@ Deploying to Heroku
 5. Connect the app with the repository (add a git remote)
 6. Push to Heroku
 7. Run database migrations (or copy the database from elsewhere)
+
+Deploying to Google Cloud Platform
+----------------------------------
+
+The GCP deployment uses OpenTofu (or Terraform) for infrastructure, Podman for
+containerization, and Google Cloud Storage for media assets.
+
+### 1. Prerequisites
+
+- [OpenTofu](https://opentofu.org/) or [Terraform](https://www.terraform.io/)
+- [Podman](https://podman.io/) or Docker
+- [Google Cloud SDK (gcloud)](https://cloud.google.com/sdk)
+
+### 2. Infrastructure Setup
+
+Initialize and apply the OpenTofu configuration in the `infra/gcp` directory:
+
+    cd infra/gcp
+    tofu init
+    tofu apply
+
+This will create the Cloud SQL instance, Cloud Run service, GCS bucket, and other necessary resources.
+
+### 3. Database Migration
+
+To import an existing database dump (e.g., from Heroku):
+
+1.  **Convert to "Clean" SQL**: Use `pg_restore` with flags to ignore ownership and privileges that won't exist on Cloud SQL.
+
+        pg_restore -O -x -f dump.sql input.dump
+
+2.  **Upload to GCS**:
+
+        gcloud storage cp dump.sql gs://your-migration-bucket/
+
+3.  **Grant Permissions**: Ensure the Cloud SQL service account can read from the bucket.
+
+        gcloud storage buckets add-iam-policy-binding gs://your-migration-bucket \
+          --member="serviceAccount:<SQL-SERVICE-ACCOUNT>" \
+          --role="roles/storage.objectViewer"
+
+    *(You can find the service account email using `gcloud sql instances describe <instance-id>`)*
+
+4.  **Run Import**:
+
+        gcloud sql import sql <instance-id> gs://your-migration-bucket/dump.sql \
+          --database=<db-name> --user=<db-user>
+
+### 4. Image Deployment
+
+A `Makefile` is provided for common deployment tasks.
+
+1.  **Authenticate with Container Registry** (one-time setup):
+
+    ```bash
+    gcloud auth configure-docker gcr.io
+    ```
+
+    *(For Podman, you may also need to run:)*
+
+    ```bash
+    gcloud auth print-access-token | podman login -u oauth2accesstoken --password-stdin https://gcr.io
+    ```
+
+2.  **Set Environment Variables**:
+
+    ```bash
+    export PROJECT_ID=your-project-id
+    export SERVICE_NAME=your-service-name
+    export ENVIRONMENT_NAME=your-environment-name
+    export REGION=your-region
+    ```
+
+3.  **Deploy** (build, push, and restart Cloud Run):
+
+    ```bash
+    make gcp-deploy
+    ```
+
+    Or run individual steps:
+
+    ```bash
+    make build       # Build the container image locally
+    make gcp-push    # Push image to GCR
+    make gcp-restart # Update the Cloud Run service
+    ```
+
+### 5. Static Files
+
+Currently, static files are served directly by the container using `dj_static.Cling`. Ensure `STATIC_URL` and `STATICFILES_STORAGE` in `settings.py` are configured appropriately (local serving is the default if GCS static configuration is commented out).

doc/DEPLOY.md

@@ -4,3 +4,6 @@
 secure_scheme_headers = {
     'X-FORWARDED-PROTO': 'https',
 }
+accesslog = '-'
+errorlog = '-'
+timeout = 120

gunicorn.conf.py

@@ -0,0 +1,38 @@
+# Example configuration - update with your actual values
+project_id         = "example-shareabouts"
+load_balancer_name = "custom-domains-abcd"
+
+# Add your domain mappings here
+# Each key is a service name, value contains domains and cloud_run_service details
+domain_mappings = {
+  # Example:
+  # shareabouts-api-dev = {
+  #   domains = ["shareaboutsapi-gcp-dev.example.com"]
+  #   cloud_run_service = {
+  #     name   = "shareabouts-api-dev"
+  #     region = "us-central1"
+  #   }
+  # }
+}
+
+# Optional: default backend for unmatched requests
+# default_backend_service = "projects/example-shareabouts/global/backendServices/default-backend"
+
+# Optional: redirect host for unmatched requests (used when default_backend_service is not set)
+# default_redirect_host = "example.com"
+
+# Optional: Legacy host rules for existing backend services not managed by this project
+# legacy_host_rules = {
+#   my-legacy-service = {
+#     hosts           = ["legacy.example.com"]
+#     path_matcher    = "legacy-example-com"
+#     backend_service = "https://www.googleapis.com/compute/v1/projects/my-project/global/backendServices/my-backend"
+#   }
+# }
+
+# Optional: Group domains into separate SSL certificates
+# domains not listed here will be grouped into a "default" certificate
+# ssl_certs = {
+#   mycity-gov    = ["suggest.mycity.gov", "suggest-staging.mycity.gov"]
+#   bikeshare-com = ["suggest.bikeshare.com"]
+# }