From 17c25912dfca888582793f3e327c21090c0dbd65 Mon Sep 17 00:00:00 2001 From: Mirco Rudolph Date: Fri, 7 Nov 2025 13:31:06 +0200 Subject: [PATCH] chore: update readme and better error logging --- climate-advisor/README.md | 32 ++++- climate-advisor/scripts/setup_database.py | 159 +++++++++++++--------- 2 files changed, 123 insertions(+), 68 deletions(-) diff --git a/climate-advisor/README.md b/climate-advisor/README.md index 28d00a015c..5926aa09bc 100644 --- a/climate-advisor/README.md +++ b/climate-advisor/README.md @@ -232,13 +232,22 @@ CC_BASE_URL=http://localhost:3000 docker run --name ca-postgres -e POSTGRES_PASSWORD=admin -e POSTGRES_DB=postgres \ -p 5432:5432 -d postgres:15 +# Fixed commands (check for user) +# give hint on actual docker name container E.g. bold_buck random name +docker exec -i bold_buck psql -U citycatalyst -d postgres -v ON_ERROR_STOP=1 -c "CREATE USER climateadvisor WITH PASSWORD 'climateadvisor';" + +docker exec -i bold_buck psql -U citycatalyst -d postgres -v ON_ERROR_STOP=1 -c "CREATE DATABASE climateadvisor OWNER climateadvisor;" + +docker exec -i bold_buck psql -U citycatalyst -d postgres -v ON_ERROR_STOP=1 -c "GRANT ALL PRIVILEGES ON DATABASE climateadvisor TO climateadvisor;" + +docker exec -i bold_buck psql -U citycatalyst -d postgres -v ON_ERROR_STOP=1 -c "ALTER USER climateadvisor CREATEDB;" + # Setup database and user -docker exec -i ca-postgres psql -U postgres -d postgres << EOF +docker exec -i ca-postgres psql -U postgres -d postgres CREATE USER climateadvisor WITH PASSWORD 'climateadvisor'; CREATE DATABASE climateadvisor OWNER climateadvisor; GRANT ALL PRIVILEGES ON DATABASE climateadvisor TO climateadvisor; ALTER USER climateadvisor CREATEDB; -EOF # Install pgvector extension docker exec ca-postgres apt update @@ -246,6 +255,19 @@ docker exec ca-postgres apt install -y postgresql-15-pgvector docker exec ca-postgres psql -U postgres -d climateadvisor -c "CREATE EXTENSION IF NOT EXISTS vector;" ``` +``` +TODO: add comments for docker exec bold_buck sh -lc "apt-get update && apt-get install + +docker exec bold_buck sh -lc "apt-get update && apt-get install -y postgresql-16-pgvector" +docker exec bold_buck sh -lc "ls /usr/share/postgresql/16/extension/vector.control" +docker exec -i bold_buck psql -U citycatalyst -d climateadvisor -c "CREATE EXTENSION IF NOT EXISTS vector;" + + +on windows: +TODO: Create additional key in .env for database setup +CA_DATABASE_URL=postgresql://climateadvisor:climateadvisor@localhost:5432/climateadvisor +``` + ### 4. Install Dependencies & Setup Database ```bash @@ -264,6 +286,12 @@ cd climate-advisor/service uvicorn app.main:app --host 0.0.0.0 --port 8080 --reload ``` +TODO: instead of uvicorn chanfe config to use docker container here + +something like this: +docker build -t hiap-app . +docker run -it --rm -p 8000:8000 --env-file .env hiap-app + ### 6. Verify Setup - **API Docs**: http://localhost:8080/docs diff --git a/climate-advisor/scripts/setup_database.py b/climate-advisor/scripts/setup_database.py index 885bf8e56d..36d822aec7 100644 --- a/climate-advisor/scripts/setup_database.py +++ b/climate-advisor/scripts/setup_database.py @@ -72,45 +72,51 @@ async def _check_database_connection() -> bool: try: import asyncpg from app.config.settings import get_settings - + settings = get_settings() - + if not settings.database_url: print("[!] Error: CA_DATABASE_URL environment variable is not set") return False - + # Convert URL for asyncpg if needed db_url = settings.database_url if db_url.startswith("postgresql+asyncpg://"): db_url = db_url.replace("postgresql+asyncpg://", "postgresql://", 1) - + # Mask password in output safe_url = db_url - if '@' in db_url: - parts = db_url.split('@') - if '//' in parts[0]: - user_pass = parts[0].split('//')[1] - if ':' in user_pass: - safe_url = db_url.replace(user_pass.split(':')[1], '***') - + if "@" in db_url: + parts = db_url.split("@") + if "//" in parts[0]: + user_pass = parts[0].split("//")[1] + if ":" in user_pass: + safe_url = db_url.replace(user_pass.split(":")[1], "***") + print(f"[*] Connecting to database: {safe_url}") - - conn = await asyncpg.connect(db_url, timeout=10) - + + conn = await asyncpg.connect(db_url, timeout=100) + # Test query version = await conn.fetchval("SELECT version()") print(f"[+] Database connection successful!") print(f" PostgreSQL version: {version.split(',')[0]}") - + await conn.close() return True - + except ImportError as e: print(f"[!] Error: Missing dependency: {e}") print(" Install with: pip install asyncpg") return False except Exception as e: - print(f"[!] Database connection failed: {e}") + print(f"[!] Database connection failed: {e.__class__.__name__}: {e}") + try: + import traceback + + traceback.print_exc() + except Exception: + pass return False @@ -119,36 +125,44 @@ async def _drop_all_tables() -> bool: try: import asyncpg from app.config.settings import get_settings - + settings = get_settings() db_url = settings.database_url - + if db_url.startswith("postgresql+asyncpg://"): db_url = db_url.replace("postgresql+asyncpg://", "postgresql://", 1) - + print("[-] Dropping all tables and extensions...") - + conn = await asyncpg.connect(db_url) - + # Drop tables with CASCADE - await conn.execute(""" + await conn.execute( + """ DROP TABLE IF EXISTS messages CASCADE; DROP TABLE IF EXISTS threads CASCADE; DROP TABLE IF EXISTS document_embeddings CASCADE; DROP TABLE IF EXISTS alembic_version CASCADE; - """) - + """ + ) + # Drop types await conn.execute("DROP TYPE IF EXISTS message_role CASCADE;") - + # Note: Not dropping vector extension as it might be used by other databases print("[+] All tables dropped successfully") - + await conn.close() return True - + except Exception as e: - print(f"[!] Error dropping tables: {e}") + print(f"[!] Error dropping tables: {e.__class__.__name__}: {e}") + try: + import traceback + + traceback.print_exc() + except Exception: + pass return False @@ -158,44 +172,50 @@ def _run_alembic_migrations() -> bool: # Change to service directory where alembic.ini is located original_dir = os.getcwd() os.chdir(SERVICE_ROOT) - + print("[*] Running Alembic migrations...") print(f" Working directory: {SERVICE_ROOT}") - + # Run alembic upgrade head result = subprocess.run( [sys.executable, "-m", "alembic", "upgrade", "head"], capture_output=True, text=True, - check=False + check=False, ) - + # Print output if result.stdout: - for line in result.stdout.strip().split('\n'): + for line in result.stdout.strip().split("\n"): if line.strip(): print(f" {line}") - + if result.returncode != 0: print(f"[!] Migration failed!") if result.stderr: print("Error output:") - for line in result.stderr.strip().split('\n'): + for line in result.stderr.strip().split("\n"): if line.strip(): print(f" {line}") return False - + print("[+] Migrations completed successfully!") - + # Change back to original directory os.chdir(original_dir) return True - + except FileNotFoundError: print("[!] Error: Alembic not found. Install with: pip install alembic") return False except Exception as e: - print(f"[!] Error running migrations: {e}") + print(f"[!] Error running migrations: {e.__class__.__name__}: {e}") + try: + import traceback + + traceback.print_exc() + except Exception: + pass return False @@ -204,62 +224,68 @@ def _show_migration_status() -> None: try: original_dir = os.getcwd() os.chdir(SERVICE_ROOT) - + print("\n[i] Current migration status:") result = subprocess.run( [sys.executable, "-m", "alembic", "current"], capture_output=True, text=True, - check=False + check=False, ) - + if result.stdout: - for line in result.stdout.strip().split('\n'): + for line in result.stdout.strip().split("\n"): if line.strip(): print(f" {line}") - + os.chdir(original_dir) - + except Exception as e: - print(f"[!] Could not get migration status: {e}") + print(f"[!] Could not get migration status: {e.__class__.__name__}: {e}") + try: + import traceback + + traceback.print_exc() + except Exception: + pass async def setup_database(drop_existing: bool = False, check_only: bool = False) -> bool: """ Set up the Climate Advisor database. - + Args: drop_existing: If True, drop all tables before running migrations check_only: If True, only check connectivity without making changes - + Returns: True if successful, False otherwise """ print("Climate Advisor Database Setup") print("=" * 50) - + # Check database connection if not await _check_database_connection(): return False - + if check_only: print("\n[+] Database connectivity check passed!") return True - + # Drop tables if requested if drop_existing: print("\n[!] WARNING: This will delete all existing data!") if not await _drop_all_tables(): return False - + # Run migrations print() if not _run_alembic_migrations(): return False - + # Show final status _show_migration_status() - + return True @@ -275,7 +301,7 @@ def main() -> None: %(prog)s --check # Check database connectivity only For more information, see: climate-advisor/README.md - """ + """, ) parser.add_argument( "--drop", @@ -288,39 +314,40 @@ def main() -> None: help="Only check database connectivity without making changes", ) args = parser.parse_args() - + # Load environment _load_env() - + # Run setup try: - success = asyncio.run(setup_database( - drop_existing=args.drop, - check_only=args.check - )) - + success = asyncio.run( + setup_database(drop_existing=args.drop, check_only=args.check) + ) + if success: print("\n" + "=" * 50) print("[+] Database setup completed successfully!") if not args.check: print("\nNext steps:") - print(" 1. Start the service: cd service && uvicorn app.main:app --reload") + print( + " 1. Start the service: cd service && uvicorn app.main:app --reload" + ) print(" 2. Visit: http://localhost:8080/docs") else: print("\n" + "=" * 50) print("[!] Database setup failed!") sys.exit(1) - + except KeyboardInterrupt: print("\n\n[!] Setup interrupted by user") sys.exit(1) except Exception as e: print(f"\n[!] Unexpected error: {e}") import traceback + traceback.print_exc() sys.exit(1) if __name__ == "__main__": main() -