Release/5.48.0

deploy/build_mcp_docs.py

@@ -31,6 +31,8 @@
     DocGroup.INVESTIGATE,
     DocGroup.BROWSE_PROFILING,
     DocGroup.TRIGGER,
+    DocGroup.SCORING,
+    DocGroup.MANAGE,
 ]
 _FALLBACK_GROUP = "Other tools"
 

deploy/testgen.dockerfile

@@ -1,4 +1,4 @@
-ARG TESTGEN_BASE_LABEL=v15
+ARG TESTGEN_BASE_LABEL=v16
 
 FROM datakitchen/dataops-testgen-base:${TESTGEN_BASE_LABEL} AS release-image
 

pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dataops-testgen"
-version = "5.33.3"
+version = "5.48.0"
 description = "DataKitchen's Data Quality DataOps TestGen"
 authors = [
     { "name" = "DataKitchen, Inc.", "email" = "info@datakitchen.io" },
@@ -41,6 +41,7 @@ dependencies = [
     "oracledb==3.4.0",
     "hdbcli==2.25.31",
     "sqlalchemy-hana==4.4.0",
+    "salesforce-cdp-connector>=1.0.19",
     "pyodbc==5.2.0",
     "psycopg2-binary==2.9.11",
     "pycryptodome==3.21",
@@ -117,6 +118,9 @@ release = [
 testgen = "testgen.__main__:cli"
 tg-patch-streamlit = "testgen.ui.scripts.patch_streamlit:patch"
 
+[project.entry-points."sqlalchemy.dialects"]
+salesforce_data360 = "testgen.common.database.salesforce_data360_dialect:SalesforceData360Dialect"
+
 [project.urls]
 "Source Code" = "https://github.com/DataKitchen/dataops-testgen"
 "Bug Tracker" = "https://github.com/DataKitchen/dataops-testgen/issues"
@@ -397,3 +401,7 @@ asset_dir = "ui/components/frontend/js"
 [[tool.streamlit.component.components]]
 name = "sidebar"
 asset_dir = "ui/components/frontend/js"
+
+[[tool.streamlit.component.components]]
+name = "feedback_widget"
+asset_dir = "ui/components/frontend/js"

testgen/__main__.py

@@ -982,7 +982,7 @@ def init_ui():
             "run",
             app_file,
             "--browser.gatherUsageStats=false",
-            f"--logger.level={'debug' if settings.IS_DEBUG else 'error'}",
+            "--logger.level=error",
             "--client.showErrorDetails=none",
             "--client.toolbarMode=minimal",
             "--server.enableStaticServing=true",

testgen/api/__init__.py

@@ -0,0 +1,12 @@
+from fastapi import APIRouter
+
+from testgen.api.app import router as _app_router
+from testgen.api.jobs import router as _jobs_router
+from testgen.api.runs import router as _runs_router
+from testgen.api.test_definitions import router as _test_definitions_router
+
+router = APIRouter(prefix="/api/v1")
+router.include_router(_app_router)
+router.include_router(_jobs_router)
+router.include_router(_runs_router)
+router.include_router(_test_definitions_router)

testgen/api/app.py

@@ -5,7 +5,7 @@
 from testgen.api.deps import db_session
 from testgen.common import version_service
 
-router = APIRouter(prefix="/api/v1", tags=["API"], dependencies=[Depends(db_session)])
+router = APIRouter(tags=["API"], dependencies=[Depends(db_session)])
 
 
 @router.get("/health")

testgen/api/deps.py

@@ -4,10 +4,14 @@
 
 from fastapi import Depends, HTTPException, Security, status
 from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+from sqlalchemy import select
 
 from testgen.common.auth import authorize_token, decode_jwt_token
 from testgen.common.models import Session, _current_session_wrapper, get_current_session
+from testgen.common.models.job_execution import PUBLIC_JOB_KEYS, JobExecution
 from testgen.common.models.project_membership import ProjectMembership
+from testgen.common.models.table_group import TableGroup
+from testgen.common.models.test_suite import TestSuite
 from testgen.common.models.user import User
 from testgen.utils.plugins import PluginHook
 
@@ -73,14 +77,25 @@ def has_project_permission(user: User, project_code: str, permission: str) -> bo
 
 # --- Resolver dependency factories ---
 # Each factory takes a permission string and returns Depends(). The entity ID
-# comes from a URL path parameter (FastAPI resolves it natively).
-# Entity not found and insufficient permission both raise the same 404
-# with a stable code/message — no variation that could leak the cause.
+# comes from a URL path parameter (FastAPI resolves it natively, including
+# UUID validation that yields a 422 for malformed inputs).
 
 _require_user = Depends(get_authorized_user)
 _not_found = api_error(404, "not_found", "Not found")
 
 
+def _check_access(entity, user: User, permission: str):
+    """Return ``entity`` if the user has ``permission`` on its project, else raise 404.
+
+    Entity-not-found and insufficient-permission both surface as the same 404
+    with a stable code/message — no variation that could leak the cause to an
+    unauthorized caller.
+    """
+    if entity and has_project_permission(user, entity.project_code, permission):
+        return entity
+    raise _not_found
+
+
 def resolve_project_code(permission: str):
     """Verify the user has ``permission`` on the project identified by ``project_code`` path param."""
     def dependency(project_code: str, user: User = _require_user) -> str:
@@ -92,45 +107,31 @@ def dependency(project_code: str, user: User = _require_user) -> str:
 
 def resolve_table_group(permission: str):
     """Resolve a TableGroup by ``table_group_id`` path param and verify project permission."""
-    from testgen.common.models.table_group import TableGroup
-
     def dependency(table_group_id: UUID, user: User = _require_user) -> TableGroup:
-        if (table_group := TableGroup.get(table_group_id)) and has_project_permission(user, table_group.project_code, permission):
-            return table_group
-        raise _not_found
+        return _check_access(TableGroup.get(table_group_id), user, permission)
     return Depends(dependency)
 
 
 def resolve_test_suite(permission: str):
     """Resolve a non-monitor TestSuite by ``test_suite_id`` path param and verify project permission."""
-    from testgen.common.models.test_suite import TestSuite
-
     def dependency(test_suite_id: UUID, user: User = _require_user) -> TestSuite:
-        if (test_suite := TestSuite.get_regular(test_suite_id)) and has_project_permission(user, test_suite.project_code, permission):
-            return test_suite
-        raise _not_found
+        return _check_access(TestSuite.get_regular(test_suite_id), user, permission)
     return Depends(dependency)
 
 
 def resolve_job(permission: str, *extra_filters):
     """Resolve a JobExecution by ``job_id`` path param and verify project permission.
 
-    Internally-submitted jobs (source='system') are never exposed via the API.
-    Extra ORM clauses are appended to the WHERE clause, e.g. to restrict by job_key.
-    Mismatches surface as the same 404 — no information leakage.
+    Only jobs whose ``job_key`` is in ``PUBLIC_JOB_KEYS`` are exposed via the API.
+    Internal kinds (score rollups, recalculations, monitor runs) are filtered out
+    by construction. Extra ORM clauses are appended to the WHERE clause to further
+    restrict by job_key when a caller wants a single kind.
     """
-    from sqlalchemy import select
-
-    from testgen.common.models.job_execution import JobExecution
-
     def dependency(job_id: UUID, user: User = _require_user) -> JobExecution:
         query = select(JobExecution).where(
             JobExecution.id == job_id,
-            JobExecution.source != "system",
+            JobExecution.job_key.in_(PUBLIC_JOB_KEYS),
             *extra_filters,
         )
-        job = get_current_session().scalars(query).first()
-        if job and has_project_permission(user, job.project_code, permission):
-            return job
-        raise _not_found
+        return _check_access(get_current_session().scalars(query).first(), user, permission)
     return Depends(dependency)

testgen/api/jobs.py

@@ -10,16 +10,17 @@
     resolve_table_group,
     resolve_test_suite,
 )
-from testgen.api.schemas import ErrorResponse, JobKey, JobListResponse, JobResponse, JobSource, JobSubmittedResponse
-from testgen.common.models.job_execution import JobExecution, JobStatus
+from testgen.api.schemas import ErrorResponse, JobListResponse, JobResponse, JobSubmittedResponse
+from testgen.common.enums import JobKey, JobSource, JobStatus
+from testgen.common.models.job_execution import PUBLIC_JOB_KEYS, JobExecution
 from testgen.common.models.table_group import TableGroup
 from testgen.common.models.test_suite import TestSuite
 
 _error_responses = {
     404: {"model": ErrorResponse, "description": "Not found"},
 }
 
-router = APIRouter(prefix="/api/v1", tags=["Jobs"], dependencies=[Depends(db_session)], responses=_error_responses)
+router = APIRouter(tags=["Jobs"], dependencies=[Depends(db_session)], responses=_error_responses)
 
 
 @router.post(
@@ -105,7 +106,7 @@ def list_jobs(
     """List job executions for a project, with optional filters and pagination."""
     items, total = JobExecution.list_for_project(
         project_code,
-        JobExecution.source != "system",
+        JobExecution.job_key.in_(PUBLIC_JOB_KEYS),
         job_key=job_key,
         status=status,
         page=page,

testgen/api/runs.py

@@ -26,7 +26,7 @@
     404: {"model": ErrorResponse, "description": "Not found"},
 }
 
-router = APIRouter(prefix="/api/v1", tags=["runs"], dependencies=[Depends(db_session)], responses=_error_responses)
+router = APIRouter(tags=["runs"], dependencies=[Depends(db_session)], responses=_error_responses)
 
 
 @router.get(

testgen/api/schemas.py

@@ -6,27 +6,11 @@
 
 from pydantic import BaseModel, field_validator
 
-from testgen.common.models.job_execution import JobStatus
+from testgen.common.enums import JobKey, JobSource, JobStatus
 
 # --- Jobs ---
 
 
-class JobKey(StrEnum):
-    run_profile = "run-profile"
-    run_tests = "run-tests"
-    run_monitors = "run-monitors"
-    run_test_generation = "run-test-generation"
-
-
-class JobSource(StrEnum):
-    api = "api"
-    ui = "ui"
-    scheduler = "scheduler"
-    mcp = "mcp"
-    cli = "cli"
-    backfill = "backfill"
-
-
 class JobSubmittedResponse(BaseModel):
     """Returned on 202 Accepted after successful job submission."""
 

testgen/api/test_definitions.py

@@ -21,7 +21,6 @@
 }
 
 router = APIRouter(
-    prefix="/api/v1",
     tags=["Test Definitions"],
     dependencies=[Depends(db_session)],
     responses=_error_responses,

testgen/commands/exec_job.py

@@ -11,9 +11,10 @@
 from uuid import UUID
 
 from testgen.commands.job_registry import JOB_DISPATCH, run_final_callbacks
+from testgen.common.enums import JobStatus
 from testgen.common.job_context import JobContext, job_context
 from testgen.common.models import database_session
-from testgen.common.models.job_execution import JobExecution, JobStatus
+from testgen.common.models.job_execution import JobExecution
 from testgen.utils import get_exception_message
 
 LOG = logging.getLogger("testgen")
@@ -35,8 +36,8 @@ def exec_job(job_execution_id: UUID) -> None:
                 LOG.error("Job execution %s not found", job_execution_id)
                 sys.exit(1)
 
-            handler = JOB_DISPATCH.get(job_exec.job_key)
-            if not handler:
+            job_config = JOB_DISPATCH.get(job_exec.job_key)
+            if not job_config:
                 job_exec.mark_interrupted(f"Unknown job key: {job_exec.job_key}")
                 return
 
@@ -48,7 +49,7 @@ def exec_job(job_execution_id: UUID) -> None:
             with database_session():
                 job_exec = JobExecution.get(job_execution_id)
                 job_context.set(JobContext(job_id=job_execution_id, source=job_exec.source))
-                handler(**job_exec.kwargs)
+                job_config.handler(**job_exec.kwargs)
 
             with database_session():
                 job_exec = JobExecution.get(job_execution_id)

testgen/commands/job_registry.py

@@ -1,7 +1,8 @@
 """Wiring between the JobExecution engine and the concrete job handlers.
 
 Two registries keyed by `job_key`:
-  - `JOB_DISPATCH`: maps a job to its handler (`exec_job` resolves this).
+  - `JOB_DISPATCH`: maps a job to its `JobConfig` (handler + per-job metadata).
+    `exec_job` and the scheduler resolve this.
   - `JOB_FINAL_CALLBACKS`: maps a job to post-terminal-transition callbacks
     (notifications, follow-up job submissions). `run_final_callbacks` iterates.
 
@@ -12,16 +13,19 @@
 
 import logging
 from collections.abc import Callable
+from dataclasses import dataclass
 
 from sqlalchemy import select
 
+from testgen.commands.run_data_cleanup import run_data_cleanup
 from testgen.commands.run_profiling import run_profiling
 from testgen.commands.run_recalculate_project_scores import run_recalculate_project_scores
 from testgen.commands.run_score_update import run_score_update
 from testgen.commands.run_test_execution import run_test_execution
 from testgen.commands.test_generation import run_test_generation
+from testgen.common.enums import JobKey, JobSource, JobStatus
 from testgen.common.models import database_session
-from testgen.common.models.job_execution import JobExecution, JobStatus
+from testgen.common.models.job_execution import JobExecution
 from testgen.common.models.profiling_run import ProfilingRun
 from testgen.common.models.test_run import TestRun
 from testgen.common.notifications.monitor_run import send_monitor_notifications
@@ -32,13 +36,31 @@
 
 FinalCallback = Callable[[JobExecution], None]
 
-JOB_DISPATCH: dict[str, Callable] = {
-    "run-profile": run_profiling,
-    "run-tests": run_test_execution,
-    "run-monitors": run_test_execution,
-    "run-test-generation": run_test_generation,
-    "run-score-update": run_score_update,
-    "recalculate-project-scores": run_recalculate_project_scores,
+
+@dataclass(frozen=True)
+class JobConfig:
+    """Per-job-key registration metadata.
+
+    `scheduler_source` is the value the scheduler tags `JobExecution.source`
+    with when it spawns this job key — ``"scheduler"`` for user-facing jobs,
+    ``"system"`` for system-internal jobs (e.g., retention cleanup). Read
+    only by the scheduler; direct `JobExecution.submit(source=...)` callers
+    (UI, follow-up enqueues, CLI) set their own source independently and do
+    not consult this field.
+    """
+
+    handler: Callable
+    scheduler_source: JobSource = JobSource.scheduler
+
+
+JOB_DISPATCH: dict[JobKey, JobConfig] = {
+    JobKey.run_profile:                JobConfig(handler=run_profiling),
+    JobKey.run_tests:                  JobConfig(handler=run_test_execution),
+    JobKey.run_monitors:               JobConfig(handler=run_test_execution),
+    JobKey.run_test_generation:        JobConfig(handler=run_test_generation),
+    JobKey.run_score_update:           JobConfig(handler=run_score_update, scheduler_source=JobSource.system),
+    JobKey.recalculate_project_scores: JobConfig(handler=run_recalculate_project_scores, scheduler_source=JobSource.system),
+    JobKey.run_data_cleanup:           JobConfig(handler=run_data_cleanup, scheduler_source=JobSource.system),
 }
 
 
@@ -91,18 +113,18 @@ def _enqueue_score_update(job_exec: JobExecution) -> None:
 
     with database_session():
         JobExecution.submit(
-            job_key="run-score-update",
+            job_key=JobKey.run_score_update,
             kwargs={
                 "parent_job_id": str(job_exec.id),
                 "parent_job_key": job_exec.job_key,
             },
-            source="system",
+            source=JobSource.system,
             project_code=job_exec.project_code,
         )
 
 
-JOB_FINAL_CALLBACKS: dict[str, list[FinalCallback]] = {
-    "run-profile": [_notify_profiling_run, _enqueue_score_update],
-    "run-tests": [_notify_test_run, _enqueue_score_update],
-    "run-monitors": [_notify_monitor_run],
+JOB_FINAL_CALLBACKS: dict[JobKey, list[FinalCallback]] = {
+    JobKey.run_profile: [_notify_profiling_run, _enqueue_score_update],
+    JobKey.run_tests: [_notify_test_run, _enqueue_score_update],
+    JobKey.run_monitors: [_notify_monitor_run],
 }