diff --git a/.env.example b/.env.example index 404a9ab2..9feaa605 100644 --- a/.env.example +++ b/.env.example @@ -4,3 +4,8 @@ QUARKUS_GOOGLE_CLOUD_STORAGE_HOST_OVERRIDE="http://127.0.0.1:9199" GCS_BUCKET_NAME="demo-bdt-dev.appspot.com" FIRESTORE_EMULATOR_HOST="127.0.0.1:8080" +# Library API Configuration +# Defaults to http://localhost:8083 (no config needed for dev) +# For production sync, set: +# LIBRARY_API_BASE_URL=https://library-api-1034049717668.us-central1.run.app + diff --git a/.github/workflows/load-library-metadata.yml b/.github/workflows/load-library-metadata.yml index 3bcfc7d0..6df235e4 100644 --- a/.github/workflows/load-library-metadata.yml +++ b/.github/workflows/load-library-metadata.yml @@ -16,20 +16,17 @@ jobs: with: python-version: "3.11" - - name: Install dependencies - working-directory: scripts - run: pip install -r requirements.txt - - name: Create GCP credentials file run: | - echo '${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}' > scripts/gcp-key.json + echo '${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}' > bin/library/gcp-key.json - - name: Run script - working-directory: scripts + - name: Run sync + working-directory: bin/library env: GOOGLE_APPLICATION_CREDENTIALS: gcp-key.json + LIBRARY_API_BASE_URL: https://library-api-1034049717668.us-central1.run.app run: | - python load-library-metadata.py + ./sync-metadata - name: Cleanup credentials - run: rm scripts/gcp-key.json + run: rm bin/library/gcp-key.json diff --git a/CLAUDE.md b/CLAUDE.md index dd543e35..c2de62c0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -167,6 +167,57 @@ Admin → builder-frontend → builder-api → Firebase (Firestore + Storage) - Learn DMN basics: https://learn-dmn-in-15-minutes.com/ - Access raw XML: Right-click → "Reopen with Text Editor" +### Library Check Metadata Sync + +The builder-api needs metadata about available library checks (from library-api). This metadata is stored in Firebase Storage and referenced from Firestore. + +**Automatic Sync (Development)**: +- Runs automatically when you start services via `devbox services up` +- Syncs from local library-api (http://localhost:8083) to Firebase emulators +- Happens after library-api starts, before builder-api starts +- Library checks will then be visible in the builder UI! + +**Manual Sync (Development)**: +```bash +# Re-sync after making library-api changes +./scripts/sync-library-metadata.sh + +# Then restart builder-api to pick up new metadata +# (In process-compose UI, restart the builder-api process) +``` + +**Production Sync** (maintainers only): +```bash +# Set production library-api URL and unset emulator variables +export LIBRARY_API_BASE_URL=https://library-api-1034049717668.us-central1.run.app +unset FIRESTORE_EMULATOR_HOST +unset GCS_BUCKET_NAME +unset QUARKUS_GOOGLE_CLOUD_STORAGE_HOST_OVERRIDE + +# Authenticate with GCP +gcloud auth application-default login + +# Run sync +./scripts/sync-library-metadata.sh +``` + +**How It Works**: +1. Fetches OpenAPI spec from library-api +2. Extracts check metadata (inputs, outputs, versions) +3. Uploads JSON to Firebase Storage +4. Updates Firestore `system/config` with storage path +5. builder-api reads this metadata on startup + +**Environment Configuration**: +- **Default**: `http://localhost:8083` (development mode - no config needed) +- **Production**: Set `LIBRARY_API_BASE_URL` to production Cloud Run URL +- Environment is inferred from URL pattern (localhost = dev, else = prod) + +**Troubleshooting**: +- **"Firebase Storage emulator not responding"**: Start emulators first (`firebase emulators:start`) +- **"library-api not responding"**: Start library-api (`cd library-api && quarkus dev`) +- **Stale metadata in builder-api**: Restart builder-api (reads metadata on startup) + ### Firebase Emulators The project uses Firebase emulators for local development: diff --git a/scripts/load-library-metadata.py b/bin/library/load-library-metadata.py similarity index 83% rename from scripts/load-library-metadata.py rename to bin/library/load-library-metadata.py index 9f1dd34c..6ce0d6f3 100644 --- a/scripts/load-library-metadata.py +++ b/bin/library/load-library-metadata.py @@ -4,16 +4,51 @@ from firebase_admin import credentials, storage, firestore import json from datetime import datetime +import os + +# ----------------------------------- +# CONFIGURATION +# ----------------------------------- + +# Default to localhost for developer-friendly setup +DEFAULT_LIBRARY_API_URL = "http://localhost:8083" +LIBRARY_API_BASE_URL = os.getenv("LIBRARY_API_BASE_URL", DEFAULT_LIBRARY_API_URL) + +# Infer production mode from URL (for versioned URL logic) +IS_PRODUCTION = not ("localhost" in LIBRARY_API_BASE_URL or "127.0.0.1" in LIBRARY_API_BASE_URL) + +# Storage bucket defaults - use dev bucket by default +DEFAULT_DEV_BUCKET = "demo-bdt-dev.appspot.com" +DEFAULT_PROD_BUCKET = "benefit-decision-toolkit-play.firebasestorage.app" +STORAGE_BUCKET = os.getenv("GCS_BUCKET_NAME", + DEFAULT_PROD_BUCKET if IS_PRODUCTION else DEFAULT_DEV_BUCKET) + +# Log configuration +print(f"========================================") +print(f"Library Metadata Sync Configuration") +print(f"========================================") +print(f"Mode: {'production' if IS_PRODUCTION else 'development'}") +print(f"Library API URL: {LIBRARY_API_BASE_URL}") +print(f"Storage Bucket: {STORAGE_BUCKET}") +print(f"========================================\n") # ----------------------------------- # INIT FIREBASE # ----------------------------------- +# Point google-cloud-storage SDK at the emulator using the existing Quarkus config variable +storage_host_override = os.getenv("QUARKUS_GOOGLE_CLOUD_STORAGE_HOST_OVERRIDE") +if storage_host_override: + os.environ["STORAGE_EMULATOR_HOST"] = storage_host_override + cred = credentials.ApplicationDefault() -firebase_admin.initialize_app(cred, { - "storageBucket": "benefit-decision-toolkit-play.firebasestorage.app" -}) +firebase_options = {"storageBucket": STORAGE_BUCKET} +if not IS_PRODUCTION: + # Emulators need an explicit project ID; production gets it from credentials + firebase_options["projectId"] = os.getenv("QUARKUS_GOOGLE_CLOUD_PROJECT_ID", "demo-bdt-dev") + +firebase_admin.initialize_app(cred, firebase_options) db = firestore.client() bucket = storage.bucket() @@ -292,7 +327,9 @@ def save_json_to_storage_and_update_firestore(json_string, firestore_doc_path): # -------------------------------------------- if __name__ == "__main__": - url = "https://library-api-1034049717668.us-central1.run.app/q/openapi.json" + url = f"{LIBRARY_API_BASE_URL}/q/openapi.json" + + print(f"Fetching OpenAPI spec from: {url}") # Send a GET request response = requests.get(url) diff --git a/scripts/requirements.txt b/bin/library/requirements.txt similarity index 100% rename from scripts/requirements.txt rename to bin/library/requirements.txt diff --git a/bin/library/sync-metadata b/bin/library/sync-metadata new file mode 100755 index 00000000..12e1a88f --- /dev/null +++ b/bin/library/sync-metadata @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" + +# Load environment from root .env if it exists +if [ -f "$PROJECT_ROOT/.env" ]; then + set -a + source "$PROJECT_ROOT/.env" + set +a +fi + +# Determine library API URL (defaults to localhost) +LIBRARY_API_URL="${LIBRARY_API_BASE_URL:-http://localhost:8083}" + +# Infer mode from URL +if [[ "$LIBRARY_API_URL" == *"localhost"* ]] || [[ "$LIBRARY_API_URL" == *"127.0.0.1"* ]]; then + MODE="development" +else + MODE="production" +fi + +echo "==========================================" +echo "Library Metadata Sync" +echo "==========================================" +echo "Mode: $MODE" +echo "Library API: $LIBRARY_API_URL" + +if [ "$MODE" = "development" ]; then + echo "Target: Firebase Emulators (Firestore + Storage)" + echo "" + echo "Prerequisites:" + echo " 1. Firebase emulators must be running" + echo " 2. library-api must be running (quarkus dev)" + echo "" + + # Check if Firebase Storage emulator is running + if ! curl -s http://localhost:9199 >/dev/null 2>&1; then + echo "ERROR: Firebase Storage emulator not responding at localhost:9199" + echo "Start emulators with: firebase emulators:start --project demo-bdt-dev --only auth,storage,firestore" + exit 1 + fi + + # Check if library-api is running + if ! curl -s "${LIBRARY_API_URL}/q/health" >/dev/null 2>&1; then + echo "ERROR: library-api not responding at ${LIBRARY_API_URL}" + echo "Start library-api with: cd library-api && quarkus dev" + exit 1 + fi + + # Check if $VENV_DIR is set and activate it + if [[ "$VENV_DIR" != "" ]]; then + if [ -f "$VENV_DIR/bin/activate" ]; then + source "$VENV_DIR/bin/activate" + echo "✓ Activated virtual environment at $VENV_DIR" + else + echo "WARNING: Virtual environment not found at $VENV_DIR" + fi + fi + + echo "✓ Firebase emulators are running" + echo "✓ library-api is running" + echo "" +else + echo "Target: Production Firebase" + echo "" + echo "Prerequisites:" + echo " - Valid Google Cloud credentials (Application Default Credentials)" + echo " - Deployed library-api at: $LIBRARY_API_URL" + echo "" + + # Check if we have GCP credentials + if ! gcloud auth application-default print-access-token >/dev/null 2>&1; then + echo "ERROR: No valid Google Cloud credentials found" + echo "Authenticate with: gcloud auth application-default login --project benefit-decision-toolkit-play" + exit 1 + fi + + echo "✓ Google Cloud credentials found" + echo "" +fi + +# Run the Python script +echo "Running metadata sync..." +pip install -q -r $SCRIPT_DIR/requirements.txt +python3 "$SCRIPT_DIR/load-library-metadata.py" + +echo "" +echo "==========================================" +echo "Metadata sync complete!" +echo "==========================================" diff --git a/builder-api/src/main/java/org/acme/service/LibraryApiService.java b/builder-api/src/main/java/org/acme/service/LibraryApiService.java index e0827c84..b08677c3 100644 --- a/builder-api/src/main/java/org/acme/service/LibraryApiService.java +++ b/builder-api/src/main/java/org/acme/service/LibraryApiService.java @@ -12,6 +12,7 @@ import org.acme.model.domain.EligibilityCheck; import org.acme.persistence.StorageService; import org.acme.persistence.FirestoreUtils; +import org.eclipse.microprofile.config.inject.ConfigProperty; import java.net.URI; import java.net.http.HttpClient; @@ -25,14 +26,36 @@ @ApplicationScoped public class LibraryApiService { + private static final String DEFAULT_LIBRARY_API_URL = "http://localhost:8083"; + @Inject private StorageService storageService; + @ConfigProperty(name = "library-api.base-url") + Optional libraryApiBaseUrl; + private List checks; + private String effectiveBaseUrl; + private boolean useVersionedUrls; @PostConstruct void init() { try { + // Determine effective base URL + effectiveBaseUrl = libraryApiBaseUrl.orElse(DEFAULT_LIBRARY_API_URL); + + // Infer environment from URL - localhost = development, else = production + boolean isProduction = !(effectiveBaseUrl.contains("localhost") || effectiveBaseUrl.contains("127.0.0.1")); + useVersionedUrls = isProduction; + + Log.info("========================================"); + Log.info("Library API Configuration"); + Log.info("========================================"); + Log.info("Base URL: " + effectiveBaseUrl); + Log.info("Mode: " + (isProduction ? "production" : "development")); + Log.info("Versioned URLs: " + (useVersionedUrls ? "enabled" : "disabled")); + Log.info("========================================"); + // Get path of most recent library schema json document Optional> configOpt = FirestoreUtils.getFirestoreDocById("system", "config"); if (configOpt.isEmpty()){ @@ -51,6 +74,7 @@ void init() { ObjectMapper mapper = new ObjectMapper(); checks = mapper.readValue(apiSchemaJson, new TypeReference>() {}); + Log.info("Loaded " + checks.size() + " library checks"); } catch (Exception e) { throw new RuntimeException("Failed to load library api metadata", e); } @@ -87,8 +111,20 @@ public EvaluationResult evaluateCheck(CheckConfig checkConfig, Map