deploy.sh

#!/usr/bin/env bash
# ===========================================================================
# CUSTOM DATA COMMONS - UNIFIED ONE-COMMAND DEPLOYER & UPDATE MANAGER
# ===========================================================================
set -euo pipefail

# Parse command line arguments
CODE_ONLY=false
INFRA_ONLY=false
FRONTEND_ONLY=false
AGENT_ONLY=false
CONFIG_ONLY=false
RESTART=false

for arg in "$@"; do
    case $arg in
        --code-only)
            CODE_ONLY=true
            ;;
        --infra-only)
            INFRA_ONLY=true
            CODE_ONLY=true
            ;;
        --frontend-only)
            FRONTEND_ONLY=true
            CODE_ONLY=true
            ;;
        --agent-only)
            AGENT_ONLY=true
            CODE_ONLY=true
            ;;
        --config-only)
            # Fast path: push config/ (branding.json, agent-config.json,
            # prompts/, skills/, assets/) to the existing config bucket and
            # refresh the running service's branding cache. No image builds,
            # no terraform, no data ingestion.
            CONFIG_ONLY=true
            ;;
        --restart)
            # Used with --config-only: force a new Cloud Run revision so the
            # agent reloads agent-config.json / prompts / skills (read only at
            # startup). Branding alone does not need this (fetched live).
            RESTART=true
            ;;
    esac
done

# Helper to retrieve active image tags from the existing Cloud Run service
get_active_image() {
    local index="$1"
    local active_image
    active_image=$(gcloud run services describe "${INSTANCE}-datacommons" \
        --region="$REGION" \
        --project="$PROJECT_ID" \
        --format="value(spec.template.spec.containers[$index].image)" 2>/dev/null || echo "")
    echo "$active_image"
}

# Color helper utilities
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0;37m' # No Color

log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }

# Early check for required CLI tools
for cmd in gcloud terraform npm python3; do
    if ! command -v "$cmd" &>/dev/null; then
        log_error "Required tool '$cmd' is not installed or not in PATH!"
        exit 1
    fi
done

# 1. Load and Validate Configuration
ENV_FILE="deploy.env"
if [ ! -f "$ENV_FILE" ]; then
    log_error "Configuration file '$ENV_FILE' not found!"
    echo "Please copy 'deploy.env.template' to 'deploy.env' and fill in your details first."
    exit 1
fi

log_info "Loading configuration from $ENV_FILE..."
# Read env variables, ignoring comments
export $(grep -v '^#' "$ENV_FILE" | xargs)

# Validation check
REQUIRED_VARS=(PROJECT_ID REGION INSTANCE DC_API_KEY MAPS_API_KEY GEMINI_API_KEY IAP_AUTHORIZED_MEMBERS)
for var in "${REQUIRED_VARS[@]}"; do
    if [ -z "${!var:-}" ] || [ "${!var}" = "YOUR_${var}" ] || [ "${!var}" = "YOUR_DC_API_KEY" ] || [ "${!var}" = "YOUR_MAPS_API_KEY" ] || [ "${!var}" = "YOUR_GEMINI_KEY" ]; then
        log_error "Required configuration variable '$var' is empty or has placeholder value in $ENV_FILE!"
        exit 1
    fi
done

# Ensure DNS compatibility for instance label
if [[ ! "$INSTANCE" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]]; then
    log_error "INSTANCE name '$INSTANCE' must be lowercase, alphanumeric, and DNS-safe."
    exit 1
fi

log_info "Configuring active gcloud project to '$PROJECT_ID'..."
gcloud config set project "$PROJECT_ID" --quiet

# --config-only: fast, deploy-free config/branding update against an existing
# instance. Syncs config/ to the bucket and nudges the agent to drop its
# branding cache, then exits — no images, terraform, secrets, or data.
if [ "$CONFIG_ONLY" = true ]; then
    CONFIG_BUCKET="${PROJECT_ID}-config"
    if ! gcloud storage buckets describe "gs://${CONFIG_BUCKET}" --project="$PROJECT_ID" &>/dev/null; then
        log_error "Config bucket 'gs://${CONFIG_BUCKET}' does not exist. Run a full deploy first."
        exit 1
    fi

    log_info "[Config-Only] Synchronizing config/ assets to GCS config bucket..."
    gcloud storage rsync config "gs://${CONFIG_BUCKET}" --recursive --project="$PROJECT_ID"
    log_success "Configuration assets synced."

    # Optional --restart: force a new Cloud Run revision so the agent reloads
    # agent-config.json / prompts / skills (read only at startup). branding.json
    # is fetched live and does not require this.
    if [ "$RESTART" = true ]; then
        log_info "[Config-Only] Forcing service restart to reload agent-config/prompts/skills..."
        if gcloud run services update "${INSTANCE}-datacommons" \
            --region="$REGION" --project="$PROJECT_ID" \
            --update-env-vars "FORCE_RESTART=$(date +%s)"; then
            log_success "Restart triggered; new revision rolling out."
        else
            log_warn "Restart request failed; agent-config may not reload until next deploy."
        fi
    fi

    # Best-effort: force the running service to refetch branding immediately
    # (otherwise it refreshes within the agent's BRAND_CACHE_TTL, default 60s).
    SERVICE_URL=$(gcloud run services describe "${INSTANCE}-datacommons" \
        --region="$REGION" --project="$PROJECT_ID" \
        --format='value(status.url)' 2>/dev/null || echo "")
    if [ -n "$SERVICE_URL" ]; then
        log_info "[Config-Only] Refreshing branding cache at ${SERVICE_URL}/agent/brand?refresh=1 ..."
        HTTP_CODE=$(curl -sS -o /dev/null -w "%{http_code}" "${SERVICE_URL}/agent/brand?refresh=1" || echo "000")
        log_info "[Config-Only] Refresh returned HTTP ${HTTP_CODE} (best-effort)."
    else
        log_warn "[Config-Only] Service URL not found; branding will refresh within the agent's cache TTL."
    fi

    log_success "[Config-Only] Done. Config changes are live (or will be within the cache TTL)."
    exit 0
fi

# 2. Enable Required APIs (Skipped in --code-only mode)
if [ "$CODE_ONLY" = false ]; then
    log_info "Enabling required Google Cloud APIs..."
    APIS=(
        run.googleapis.com
        sqladmin.googleapis.com
        secretmanager.googleapis.com
        cloudbuild.googleapis.com
        artifactregistry.googleapis.com
        storage.googleapis.com
        compute.googleapis.com
        apikeys.googleapis.com
        generativelanguage.googleapis.com
    )
    gcloud services enable "${APIS[@]}" --project="$PROJECT_ID"
    log_success "All required APIs enabled successfully."
else
    log_info "[Fast-Track] Skipping Google Cloud API enablement check."
fi

# 3. Bootstrap State Bucket & Artifact Registry (Skipped in --code-only mode)
STATE_BUCKET="${PROJECT_ID}-tfstate"
AR_REPO="custom-dc"
if [ "$CODE_ONLY" = false ]; then
    if ! gcloud storage buckets describe "gs://${STATE_BUCKET}" --project="$PROJECT_ID" &>/dev/null; then
        log_info "Creating Terraform state bucket 'gs://${STATE_BUCKET}'..."
        gcloud storage buckets create "gs://${STATE_BUCKET}" --project="$PROJECT_ID" --location="$REGION" --uniform-bucket-level-access
        gcloud storage buckets update "gs://${STATE_BUCKET}" --project="$PROJECT_ID" --versioning
        log_success "State bucket created."
    else
        log_info "Terraform state bucket 'gs://${STATE_BUCKET}' already exists."
    fi

    if ! gcloud artifacts repositories describe "$AR_REPO" --location="$REGION" --project="$PROJECT_ID" &>/dev/null; then
        log_info "Creating Artifact Registry repository '$AR_REPO' in '$REGION'..."
        gcloud artifacts repositories create "$AR_REPO" \
            --repository-format=docker \
            --location="$REGION" \
            --description="Docker repository for Custom Data Commons" \
            --project="$PROJECT_ID"
        log_success "Artifact Registry repository created."
    else
        log_info "Artifact Registry repository '$AR_REPO' already exists."
    fi
else
    log_info "[Fast-Track] Skipping GCS state bucket and Artifact Registry checks."
fi

# 4. Secret Manager Setup (Skipped in --code-only mode)
if [ "$CODE_ONLY" = false ]; then
    create_secret_if_missing() {
        local secret_id="$1"
        if ! gcloud secrets describe "$secret_id" --project="$PROJECT_ID" &>/dev/null; then
            log_info "Creating secret '$secret_id' in Secret Manager..."
            gcloud secrets create "$secret_id" --replication-policy="automatic" --project="$PROJECT_ID"
        fi
    }

    add_secret_version_if_changed() {
        local secret_id="$1"
        local secret_val="$2"
        if gcloud secrets versions describe latest --secret="$secret_id" --project="$PROJECT_ID" &>/dev/null; then
            local current_val
            # Fetch current value, suppressing errors if inaccessible
            current_val=$(gcloud secrets versions access latest --secret="$secret_id" --project="$PROJECT_ID" 2>/dev/null || echo "")
            if [ "$current_val" = "$secret_val" ]; then
                log_info "Secret '$secret_id' value matches latest version. Skipping upload."
                return
            fi
        fi
        log_info "Uploading new version for secret '$secret_id'..."
        echo -n "$secret_val" | gcloud secrets versions add "$secret_id" --data-file=- --project="$PROJECT_ID" >/dev/null
    }

    # Database password creation (generated randomly once if missing)
    DB_PASS_SECRET="${INSTANCE}-db-pass"
    create_secret_if_missing "$DB_PASS_SECRET"
    if ! gcloud secrets versions describe latest --secret="$DB_PASS_SECRET" --project="$PROJECT_ID" &>/dev/null; then
        log_info "Generating secure random database password..."
        RAND_PASS=$(LC_ALL=C tr -dc 'A-Za-z0-9' </dev/urandom | head -c 32 || true)
        echo -n "$RAND_PASS" | gcloud secrets versions add "$DB_PASS_SECRET" --data-file=- --project="$PROJECT_ID" >/dev/null
    fi

    # DC API Key (idempotent)
    DC_SECRET="${INSTANCE}-dc-api-key"
    create_secret_if_missing "$DC_SECRET"
    add_secret_version_if_changed "$DC_SECRET" "$DC_API_KEY"

    # Maps API Key (idempotent)
    MAPS_SECRET="${INSTANCE}-maps-api-key"
    create_secret_if_missing "$MAPS_SECRET"
    add_secret_version_if_changed "$MAPS_SECRET" "$MAPS_API_KEY"

    # Gemini API Key (idempotent JSON array)
    GEMINI_SECRET="${INSTANCE}-gemini-api-keys"
    create_secret_if_missing "$GEMINI_SECRET"
    GEMINI_JSON="[\"${GEMINI_API_KEY}\"]"
    add_secret_version_if_changed "$GEMINI_SECRET" "$GEMINI_JSON"

    log_success "All secrets securely stored in Secret Manager."
else
    log_info "[Fast-Track] Skipping Secret Manager checks."
fi

# 5. Build and Stage React UI Frontend
if [ "$INFRA_ONLY" = false ] && [ "$AGENT_ONLY" = false ]; then
    log_info "Compiling React UI production bundle..."
    cd ui
    npm ci --registry=https://registry.npmjs.org/
    npm run build
    cd ..

    log_info "Syncing compiled static assets to container build staging directory..."
    rm -rf image/dist/*
    cp -r ui/dist/* image/dist/
    log_success "Frontend assets staged successfully."
else
    log_info "[Surgical-Build] Skipping React UI compilation."
fi

# 6. Build and Push Container Images via Cloud Build
IMAGE_TAG=$(git rev-parse --short HEAD 2>/dev/null || date +%s)
SERVICES_IMAGE=""
AGENT_IMAGE=""

if [ "$INFRA_ONLY" = true ]; then
    log_info "[Surgical-Build] Skipping all container builds. Retrieving active images from Cloud Run..."
    SERVICES_IMAGE=$(get_active_image 0)
    AGENT_IMAGE=$(get_active_image 1)
    if [ -z "$SERVICES_IMAGE" ] || [ -z "$AGENT_IMAGE" ]; then
        log_error "Could not retrieve active container images from the cloud! Please run a full deployment first."
        exit 1
    fi
elif [ "$FRONTEND_ONLY" = true ]; then
    log_info "[Surgical-Build] Frontend-only mode. Building only Services Overlay..."
    SERVICES_IMAGE="${REGION}-docker.pkg.dev/${PROJECT_ID}/${AR_REPO}/services:${IMAGE_TAG}"
    gcloud builds submit --tag="$SERVICES_IMAGE" --project="$PROJECT_ID" image || { log_error "Services container build failed!"; exit 1; }
    
    AGENT_IMAGE=$(get_active_image 1)
    if [ -z "$AGENT_IMAGE" ]; then
        log_error "Could not retrieve active Agent container image from the cloud! Please run a full deployment first."
        exit 1
    fi
elif [ "$AGENT_ONLY" = true ]; then
    log_info "[Surgical-Build] Agent-only mode. Building only Agent Sidecar..."
    AGENT_IMAGE="${REGION}-docker.pkg.dev/${PROJECT_ID}/${AR_REPO}/agent:${IMAGE_TAG}"
    gcloud builds submit --tag="$AGENT_IMAGE" --project="$PROJECT_ID" agent || { log_error "Agent sidecar container build failed!"; exit 1; }
    
    SERVICES_IMAGE=$(get_active_image 0)
    if [ -z "$SERVICES_IMAGE" ]; then
        log_error "Could not retrieve active Services container image from the cloud! Please run a full deployment first."
        exit 1
    fi
else
    log_info "Submitting container builds concurrently to Google Cloud Build (Tag: $IMAGE_TAG)..."
    AGENT_IMAGE="${REGION}-docker.pkg.dev/${PROJECT_ID}/${AR_REPO}/agent:${IMAGE_TAG}"
    log_info "Starting background build for Agent Sidecar..."
    gcloud builds submit --tag="$AGENT_IMAGE" --project="$PROJECT_ID" agent &
    AGENT_PID=$!

    SERVICES_IMAGE="${REGION}-docker.pkg.dev/${PROJECT_ID}/${AR_REPO}/services:${IMAGE_TAG}"
    log_info "Starting background build for Services Overlay..."
    gcloud builds submit --tag="$SERVICES_IMAGE" --project="$PROJECT_ID" image &
    SERVICES_PID=$!

    log_info "Waiting for container builds to complete..."
    wait $AGENT_PID || { log_error "Agent sidecar container build failed!"; exit 1; }
    wait $SERVICES_PID || { log_error "Services container build failed!"; exit 1; }
fi

log_success "Resolved Container Images:"
echo "  Agent:    $AGENT_IMAGE"
echo "  Services: $SERVICES_IMAGE"

# 7. Create & Seed Configuration Bucket (Skipped in --code-only mode)
CONFIG_BUCKET="${PROJECT_ID}-config"
if [ "$CODE_ONLY" = false ]; then
    if ! gcloud storage buckets describe "gs://${CONFIG_BUCKET}" --project="$PROJECT_ID" &>/dev/null; then
        log_info "Creating GCS configuration bucket 'gs://${CONFIG_BUCKET}'..."
        gcloud storage buckets create "gs://${CONFIG_BUCKET}" --project="$PROJECT_ID" --location="$REGION" --uniform-bucket-level-access
        CORS_FILE=$(mktemp)
        echo '[{"origin": ["*"], "method": ["GET", "OPTIONS"], "responseHeader": ["Content-Type"], "maxAgeSeconds": 3600}]' > "$CORS_FILE"
        gcloud storage buckets update "gs://${CONFIG_BUCKET}" --cors-file="$CORS_FILE" --project="$PROJECT_ID"
        rm -f "$CORS_FILE"
        log_success "Config bucket created."
    else
        log_info "GCS configuration bucket 'gs://${CONFIG_BUCKET}' already exists."
    fi

    log_info "Synchronizing config/ assets to GCS config bucket..."
    gcloud storage rsync config "gs://${CONFIG_BUCKET}" --recursive --project="$PROJECT_ID"
    log_success "Configuration assets seeded successfully."
else
    log_info "[Fast-Track] Skipping config bucket synchronization."
fi

# 8. Generate tfvars configuration
TFVARS_FILE="deploy/terraform-custom-datacommons/${INSTANCE}.tfvars"
log_info "Generating Terraform variable overrides file: $TFVARS_FILE"

IAP_LIST=""
IFS=',' read -ra ADDR <<< "$IAP_AUTHORIZED_MEMBERS"
for member in "${ADDR[@]}"; do
    if [ -n "$IAP_LIST" ]; then IAP_LIST="${IAP_LIST}, "; fi
    IAP_LIST="${IAP_LIST}\"${member}\""
done

# Perform 100% portable, OS-independent token replacement via Python
python3 -c "
import sys
with open(sys.argv[1], 'r') as f:
    content = f.read()
replacements = {
    'REPLACE_PROJECT_ID': sys.argv[2],
    'REPLACE_REGION': sys.argv[3],
    'REPLACE_INSTANCE': sys.argv[4],
    'REPLACE_SERVICES_IMAGE': sys.argv[5],
    'REPLACE_AGENT_IMAGE': sys.argv[6],
    'REPLACE_OUTPUT_DIR': '',
    'REPLACE_INPUT_DIR': '',
    'REPLACE_IAP_MEMBERS': sys.argv[7]
}
for k, v in replacements.items():
    content = content.replace(k, v)
with open(sys.argv[8], 'w') as f:
    f.write(content)
" deploy/terraform-custom-datacommons/new-instance.tfvars.sample \
  "$PROJECT_ID" "$REGION" "$INSTANCE" "$SERVICES_IMAGE" "$AGENT_IMAGE" "$IAP_LIST" "$TFVARS_FILE"

log_success "tfvars file generated cleanly."

# 9. Run Terraform Pipeline
cd deploy/terraform-custom-datacommons/modules

log_info "Initializing Terraform backend..."
terraform init \
    -backend-config="bucket=${STATE_BUCKET}" \
    -backend-config="prefix=custom-datacommons/${INSTANCE}" \
    -reconfigure

if [ "$CODE_ONLY" = false ]; then
    log_info "Provisioning data-plane infrastructure resources..."
    terraform apply -auto-approve \
        -var-file="../${INSTANCE}.tfvars" \
        -target=google_sql_database_instance.dc \
        -target=google_sql_database.dc \
        -target=google_sql_user.dc \
        -target=google_storage_bucket.data \
        -target=google_service_account.datacommons \
        -target=google_project_iam_member.secret_accessor \
        -target=google_project_iam_member.cloudsql_client \
        -target=google_project_iam_member.log_writer \
        -target=google_project_iam_member.metric_writer \
        -target=google_project_iam_member.trace_agent \
        -target=google_storage_bucket_iam_member.config_reader \
        -target=google_storage_bucket_iam_member.data_reader \
        -target=google_cloud_run_v2_job.data_ingest

    # Stage data and execute ingest job
    DATA_BUCKET="${INSTANCE}-data-${PROJECT_ID}"
    log_info "Staging sample dataset CSV files to GCS data bucket 'gs://${DATA_BUCKET}/input/'..."
    gcloud storage cp -r ../../../sample-data/* "gs://${DATA_BUCKET}/input/" --project="$PROJECT_ID"

    log_info "Triggering data-plane SQL database ingestion job..."
    gcloud run jobs execute "${INSTANCE}-data-ingest" \
        --region="$REGION" --project="$PROJECT_ID" --wait
else
    log_info "[Fast-Track] Skipping data-plane provisioning and database ingestion."
fi

# Deploy services (Updates Cloud Run to point to new image tags)
log_info "Provisioning and updating service container configurations..."
terraform apply -auto-approve -var-file="../${INSTANCE}.tfvars"

# Retrieve final service endpoint URL
SERVICE_URL=$(terraform output -raw service_url)
cd ../../..

# 10. Run Verification Smoke Tests
if [ "$CODE_ONLY" = false ]; then
    log_info "Temporarily disabling Identity-Aware Proxy (IAP) for validation..."
    gcloud beta run services update "${INSTANCE}-datacommons" \
        --no-iap \
        --region="$REGION" \
        --project="$PROJECT_ID"

    log_info "Running automated post-deployment smoke tests against public URL..."
    # Suppress errors if the smoke test fails, so we always re-enable IAP
    python3 /Users/suryaveer/.gemini/jetski/scratch/smoke_test.py "$SERVICE_URL" || log_warn "Smoke tests encountered failures."

    log_info "Re-enabling Identity-Aware Proxy (IAP) on Cloud Run service..."
    gcloud beta run services update "${INSTANCE}-datacommons" \
        --iap \
        --region="$REGION" \
        --project="$PROJECT_ID"
else
    log_info "[Fast-Track] Skipping IAP toggles and smoke tests."
fi

echo -e "\n==========================================================================="
log_success "CONGRATULATIONS! CUSTOM DATA COMMONS UPDATED SUCCESSFULLY!"
echo -e "==========================================================================="
echo -e "Instance Display Name:  ${INSTANCE} Data Commons"
echo -e "Service Web Endpoint:   ${GREEN}${SERVICE_URL}${NC}"
echo -e "Access Security:        IAP Protected"
echo -e "Authorized Groups/Users:"
IFS=',' read -ra ADDR <<< "$IAP_AUTHORIZED_MEMBERS"
for member in "${ADDR[@]}"; do
    echo -e "  - ${member}"
done
echo -e "===========================================================================\n"