feat: support standalone install with embedded postgres

Author: aarthy-dk

pyproject.toml

@@ -87,6 +87,10 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
+standalone = [
+    "pixeltable-pgserver>=0.5.1",
+]
+
 dev = [
     "invoke==2.2.0",
     "ruff==0.4.1",

testgen/__main__.py

@@ -1,10 +1,16 @@
+import base64
+import importlib
 import logging
 import os
+import platform
+import secrets
 import signal
 import subprocess
 import sys
 from dataclasses import dataclass, field
 from datetime import UTC, datetime, timedelta
+from importlib.metadata import version as pkg_version
+from pathlib import Path
 
 import click
 from click.core import Context
@@ -42,6 +48,13 @@
     get_tg_schema,
     version_service,
 )
+from testgen.common.standalone_postgres import (
+    STANDALONE_URI_ENV_VAR,
+    get_home_dir as get_testgen_home,
+    get_server_uri,
+    is_standalone_mode,
+    start_server as start_standalone_postgres,
+)
 from testgen.common.models import with_database_session
 from testgen.common.models.profiling_run import ProfilingRun
 from testgen.common.models.settings import PersistedSetting
@@ -99,19 +112,23 @@ def invoke(self, ctx: Context):
 )
 @click.pass_context
 def cli(ctx: Context, verbose: bool):
+    if is_standalone_mode():
+        start_standalone_postgres()
+
     if verbose:
         configure_logging(level=logging.DEBUG)
     else:
         configure_logging(level=logging.INFO)
 
     ctx.obj = Configuration(verbose=verbose)
-    status_ok, message = docker_service.check_basic_configuration()
-    if not status_ok:
-        click.secho(message, fg="red")
-        sys.exit(1)
+    if not is_standalone_mode() and ctx.invoked_subcommand != "standalone-setup":
+        status_ok, message = docker_service.check_basic_configuration()
+        if not status_ok:
+            click.secho(message, fg="red")
+            sys.exit(1)
 
     if (
-        ctx.invoked_subcommand not in ["run-app", "ui", "setup-system-db", "upgrade-system-version", "quick-start"]
+        ctx.invoked_subcommand not in ["run-app", "ui", "setup-system-db", "upgrade-system-version", "quick-start", "standalone-setup"]
         and not is_db_revision_up_to_date()
     ):
         click.secho("The system database schema is outdated. Automatically running the following command:", fg="red")
@@ -472,6 +489,110 @@ def quick_start(
     click.echo("Quick start has successfully finished.")
 
 
+@cli.command("standalone-setup", help="Set up TestGen for standalone use with embedded PostgreSQL (no Docker required).")
+@click.option("--username", prompt="Admin username", default="admin", help="Username for the TestGen web UI.")
+@click.option(
+    "--password", prompt="Admin password", hide_input=True, confirmation_prompt=True,
+    default="testgen", help="Password for the TestGen web UI.",
+)
+def setup_standalone(username: str, password: str):
+    config_dir = get_testgen_home()
+    config_path = config_dir / "config.env"
+
+    if config_path.exists():
+        if not click.confirm(f"Config already exists at {config_path}. Overwrite?"):
+            click.echo("Aborted.")
+            return
+
+    # Generate secrets (same approach as dk-installer)
+    def generate_secret(length: int = 12) -> str:
+        alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+        return "".join(secrets.choice(alphabet) for _ in range(length))
+
+    jwt_key = base64.b64encode(secrets.token_bytes(32)).decode()
+    decrypt_salt = generate_secret()
+    decrypt_password = generate_secret()
+    log_dir = str(config_dir / "log")
+
+    config_dir.mkdir(parents=True, exist_ok=True)
+
+    config_lines = [
+        "# TestGen standalone configuration",
+        "# Generated by: testgen standalone-setup",
+        "",
+        "# Standalone mode (embedded PostgreSQL)",
+        "TG_STANDALONE_MODE=yes",
+        "",
+        "# UI credentials",
+        f"TESTGEN_USERNAME={username}",
+        f"TESTGEN_PASSWORD={password}",
+        "",
+        "# Encryption keys",
+        f"TG_DECRYPT_SALT={decrypt_salt}",
+        f"TG_DECRYPT_PASSWORD={decrypt_password}",
+        f"TG_JWT_HASHING_KEY={jwt_key}",
+        "",
+        "# Logging",
+        f"TESTGEN_LOG_FILE_PATH={log_dir}",
+        "",
+        "# Analytics",
+        "TG_ANALYTICS=yes",
+        "",
+        "# Trust target database certificates (for SQL Server, etc.)",
+        "TG_TARGET_DB_TRUST_SERVER_CERTIFICATE=yes",
+        "TG_EXPORT_TO_OBSERVABILITY_VERIFY_SSL=no",
+    ]
+    config_path.write_text("\n".join(config_lines) + "\n")
+    click.echo(f"Config written to {config_path}")
+
+    # Reload settings — the module was already evaluated at import time
+    # before the config file existed.  Reloading re-reads the new file
+    # and re-evaluates all module-level variables.
+    importlib.reload(settings)
+
+    # Patch Streamlit to support editable-install component resolution
+    click.echo("Patching Streamlit...")
+    from testgen.ui.scripts.patch_streamlit import patch as patch_streamlit
+    patch_streamlit(dev=True)
+
+    # Start embedded PostgreSQL (standalone mode is now active via config)
+    start_standalone_postgres()
+
+    # Initialize the database
+    click.echo("Initializing database...")
+    run_launch_db_config(delete_db=False)
+
+    # Send analytics event for pip install tracking
+    try:
+        from testgen.common.mixpanel_service import MixpanelService
+
+        mp = MixpanelService()
+        mp.send_event(
+            "standalone_setup",
+            username=username,
+            install_type="standalone",
+            version=pkg_version("dataops-testgen"),
+            python_info=f"{platform.python_implementation()} {platform.python_version()}",
+            **{"$os": platform.system()},
+            os_version=platform.release(),
+            os_arch=platform.machine(),
+        )
+    except Exception:  # noqa: S110
+        pass
+
+    click.echo("")
+    click.echo(click.style("TestGen is ready!", fg="green", bold=True))
+    click.echo("")
+    click.echo("  To load demo data (optional):")
+    click.echo("    testgen quick-start")
+    click.echo("")
+    click.echo("  Start the application:")
+    click.echo("    testgen run-app")
+    click.echo("")
+    click.echo("  Then open http://localhost:8501 in your browser.")
+    click.echo(f"  Log in with username: {username}")
+
+
 @cli.command("setup-system-db", help="Use to initialize the TestGen system database.")
 @click.option(
     "--delete-db",
@@ -728,6 +849,15 @@ def init_ui():
     init_ui()
 
     app_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "ui/app.py")
+
+    # In standalone mode, pass the pgserver URI to the Streamlit subprocess
+    # so it can connect without acquiring the pgserver file lock.
+    child_env = {**os.environ, "TG_JOB_SOURCE": "UI"}
+    if is_standalone_mode():
+        server_uri = get_server_uri()
+        if server_uri:
+            child_env = {**os.environ, "TG_JOB_SOURCE": "UI", STANDALONE_URI_ENV_VAR: server_uri}
+
     process= subprocess.Popen(
         [  # noqa: S607
             "streamlit",
@@ -742,7 +872,7 @@ def init_ui():
             "--",
             f"{'--debug' if settings.IS_DEBUG else ''}",
         ],
-        env={**os.environ, "TG_JOB_SOURCE": "UI"}
+        env=child_env,
     )
     def term_ui(signum, _):
         LOG.info(f"Sending termination signal {signum} to Testgen UI")

testgen/commands/run_launch_db_config.py

@@ -4,6 +4,7 @@
 from testgen import settings
 from testgen.common import create_database, execute_db_queries
 from testgen.common.credentials import get_tg_db, get_tg_schema
+from testgen.common.standalone_postgres import get_home_dir, is_standalone_mode
 from testgen.common.database.database_service import get_queries_for_command
 from testgen.common.encrypt import EncryptText, encrypt_ui_password
 from testgen.common.models import with_database_session
@@ -22,6 +23,14 @@ def _get_latest_revision_number():
 def _get_params_mapping() -> dict:
     ui_user_encrypted_password = encrypt_ui_password(settings.PASSWORD)
 
+    project_host = settings.PROJECT_DATABASE_HOST
+    project_user = settings.PROJECT_DATABASE_USER
+    project_password = settings.PROJECT_DATABASE_PASSWORD
+    if is_standalone_mode():
+        project_host = str(get_home_dir() / "pgdata")
+        project_user = "postgres"
+        project_password = ""
+
     return {
         "UI_USER_NAME": settings.USERNAME,
         "UI_USER_USERNAME": settings.USERNAME,
@@ -33,10 +42,10 @@ def _get_params_mapping() -> dict:
         "SQL_FLAVOR": settings.PROJECT_SQL_FLAVOR,
         "PROJECT_NAME": settings.PROJECT_NAME,
         "PROJECT_DB": settings.PROJECT_DATABASE_NAME,
-        "PROJECT_USER": settings.PROJECT_DATABASE_USER,
+        "PROJECT_USER": project_user,
         "PROJECT_PORT": settings.PROJECT_DATABASE_PORT,
-        "PROJECT_HOST": settings.PROJECT_DATABASE_HOST,
-        "PROJECT_PW_ENCRYPTED": EncryptText(settings.PROJECT_DATABASE_PASSWORD),
+        "PROJECT_HOST": project_host,
+        "PROJECT_PW_ENCRYPTED": EncryptText(project_password),
         "PROJECT_HTTP_PATH": "",
         "PROJECT_SERVICE_ACCOUNT_KEY": "",
         "PROJECT_SCHEMA": settings.PROJECT_DATABASE_SCHEMA,

testgen/commands/run_quick_start.py

@@ -8,6 +8,7 @@
 
 from testgen import settings
 from testgen.commands.run_launch_db_config import get_app_db_params_mapping, run_launch_db_config
+from testgen.common.standalone_postgres import get_home_dir, is_standalone_mode
 from testgen.commands.test_generation import run_monitor_generation
 from testgen.common.credentials import get_tg_schema
 from testgen.common.database.database_service import (
@@ -93,14 +94,22 @@ def _prepare_connection_to_target_database(params_mapping):
 
 
 def _get_settings_params_mapping() -> dict:
+    host = settings.PROJECT_DATABASE_HOST
+    admin_user = settings.DATABASE_ADMIN_USER
+    admin_password = settings.DATABASE_ADMIN_PASSWORD
+    if is_standalone_mode():
+        host = str(get_home_dir() / "pgdata")
+        admin_user = "postgres"
+        admin_password = ""
+
     return {
-        "TESTGEN_ADMIN_USER": settings.DATABASE_ADMIN_USER,
-        "TESTGEN_ADMIN_PASSWORD": settings.DATABASE_ADMIN_PASSWORD,
+        "TESTGEN_ADMIN_USER": admin_user,
+        "TESTGEN_ADMIN_PASSWORD": admin_password,
         "SCHEMA_NAME": get_tg_schema(),
         "PROJECT_DB": settings.PROJECT_DATABASE_NAME,
         "PROJECT_SCHEMA": settings.PROJECT_DATABASE_SCHEMA,
         "PROJECT_KEY": settings.PROJECT_KEY,
-        "PROJECT_DB_HOST": settings.PROJECT_DATABASE_HOST,
+        "PROJECT_DB_HOST": host,
         "PROJECT_DB_PORT": settings.PROJECT_DATABASE_PORT,
         "SQL_FLAVOR": settings.PROJECT_SQL_FLAVOR,
     }

testgen/common/database/database_service.py

@@ -32,6 +32,7 @@
     SQLFlavor,
     resolve_connection_params,
 )
+from testgen.common.standalone_postgres import get_connection_string as get_standalone_connection_string, is_standalone_mode
 from testgen.common.read_file import get_template_files
 from testgen.utils import get_exception_message
 
@@ -370,16 +371,27 @@ def _init_app_db_connection(
         engine = engine_cache.app_db
 
     if not engine:
-        user = user_override if is_admin else get_tg_username()
-        password = password_override if (is_admin or password_override is not None) else get_tg_password()
+        if is_standalone_mode():
+            connection_string = get_standalone_connection_string(database_name)
+        else:
+            user = user_override if is_admin else get_tg_username()
+            password = password_override if (is_admin or password_override is not None) else get_tg_password()
 
-        # STANDARD FORMAT: flavor://username:password@host:port/database
-        connection_string = (
-            f"postgresql://{user}:{quote_plus(password)}@{get_tg_host()}:{get_tg_port()}/{database_name}"
-        )
+            # STANDARD FORMAT: flavor://username:password@host:port/database
+            connection_string = (
+                f"postgresql://{user}:{quote_plus(password)}@{get_tg_host()}:{get_tg_port()}/{database_name}"
+            )
         try:
-            engine: Engine = create_engine(connection_string, connect_args={"connect_timeout": 3600})
-            engine_cache.app_db = engine
+            engine: Engine = create_engine(
+                connection_string,
+                connect_args={
+                    "connect_timeout": 3600,
+                    # Force UTC so TIMESTAMP-without-tz inserts aren't silently shifted.
+                    "options": "-c TimeZone=UTC",
+                },
+            )
+            if user_type == "normal":
+                engine_cache.app_db = engine
 
         except SQLAlchemyError as e:
             raise ValueError(f"Failed to create engine for App database '{database_name}' (User type = {user_type})") from e

testgen/common/database/flavor/postgresql_flavor_service.py

@@ -1,6 +1,15 @@
+from urllib.parse import quote_plus
+
+from testgen.common.database.flavor.flavor_service import ResolvedConnectionParams
 from testgen.common.database.flavor.redshift_flavor_service import RedshiftFlavorService
 
 
 class PostgresqlFlavorService(RedshiftFlavorService):
 
     escaped_underscore = "\\_"
+
+    def get_connection_string_from_fields(self, params: ResolvedConnectionParams) -> str:
+        if params.host.startswith("/"):
+            # Unix socket path — use query-param format for psycopg2
+            return f"{self.url_scheme}://{params.username}:{quote_plus(params.password)}@/{params.dbname}?host={params.host}"
+        return super().get_connection_string_from_fields(params)

testgen/common/logs.py

@@ -29,16 +29,18 @@ def configure_logging(
         logger.addHandler(console_handler)
 
         if settings.LOG_TO_FILE:
-            os.makedirs(settings.LOG_FILE_PATH, exist_ok=True)
-
-            file_handler = ConcurrentTimedRotatingFileHandler(
-                get_log_full_path(),
-                when="MIDNIGHT",
-                interval=1,
-                backupCount=int(settings.LOG_FILE_MAX_QTY),
-            )
-            file_handler.setFormatter(formatter)
-            logger.addHandler(file_handler)
+            try:
+                os.makedirs(settings.LOG_FILE_PATH, exist_ok=True)
+                file_handler = ConcurrentTimedRotatingFileHandler(
+                    get_log_full_path(),
+                    when="MIDNIGHT",
+                    interval=1,
+                    backupCount=int(settings.LOG_FILE_MAX_QTY),
+                )
+                file_handler.setFormatter(formatter)
+                logger.addHandler(file_handler)
+            except OSError:
+                logger.warning("Cannot write logs to %s — file logging disabled", settings.LOG_FILE_PATH)
 
 
 def get_log_full_path() -> str:

testgen/common/models/__init__.py

@@ -5,9 +5,7 @@
 import urllib.parse
 
 from sqlalchemy import create_engine
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import Session as SQLAlchemySession
-from sqlalchemy.orm import sessionmaker
+from sqlalchemy.orm import DeclarativeBase, Session as SQLAlchemySession, sessionmaker
 
 from testgen import settings
 
@@ -19,11 +17,17 @@
     echo=False,
     connect_args={
         "application_name": platform.node(),
-        "options": f"-csearch_path={settings.DATABASE_SCHEMA}",
+        # TimeZone=UTC so TIMESTAMP (no-tz) columns store aware UTC datetimes as-is.
+        # Without this, pgserver inherits the OS TZ and silently shifts
+        # timestamps on insert, which make_json_safe then re-reads as UTC.
+        "options": f"-csearch_path={settings.DATABASE_SCHEMA} -c TimeZone=UTC",
     },
 )
 
-Base = declarative_base()
+class Base(DeclarativeBase):
+    # Allow legacy Column() + type-hint patterns without Mapped[].
+    # Can be removed once all models use Mapped[] annotations.
+    __allow_unmapped__ = True
 
 Session = sessionmaker(
     engine,