Skip to content

Commit 0ded486

Browse files
Merge pull request #68 from DataKitchen/release/5.48.0
Release/5.48.0
2 parents ab0b3f5 + e474fe7 commit 0ded486

337 files changed

Lines changed: 26823 additions & 1443 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

deploy/build_mcp_docs.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@
3131
DocGroup.INVESTIGATE,
3232
DocGroup.BROWSE_PROFILING,
3333
DocGroup.TRIGGER,
34+
DocGroup.SCORING,
35+
DocGroup.MANAGE,
3436
]
3537
_FALLBACK_GROUP = "Other tools"
3638

deploy/testgen.dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
ARG TESTGEN_BASE_LABEL=v15
1+
ARG TESTGEN_BASE_LABEL=v16
22

33
FROM datakitchen/dataops-testgen-base:${TESTGEN_BASE_LABEL} AS release-image
44

pyproject.toml

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta"
88

99
[project]
1010
name = "dataops-testgen"
11-
version = "5.33.3"
11+
version = "5.48.0"
1212
description = "DataKitchen's Data Quality DataOps TestGen"
1313
authors = [
1414
{ "name" = "DataKitchen, Inc.", "email" = "info@datakitchen.io" },
@@ -41,6 +41,7 @@ dependencies = [
4141
"oracledb==3.4.0",
4242
"hdbcli==2.25.31",
4343
"sqlalchemy-hana==4.4.0",
44+
"salesforce-cdp-connector>=1.0.19",
4445
"pyodbc==5.2.0",
4546
"psycopg2-binary==2.9.11",
4647
"pycryptodome==3.21",
@@ -117,6 +118,9 @@ release = [
117118
testgen = "testgen.__main__:cli"
118119
tg-patch-streamlit = "testgen.ui.scripts.patch_streamlit:patch"
119120

121+
[project.entry-points."sqlalchemy.dialects"]
122+
salesforce_data360 = "testgen.common.database.salesforce_data360_dialect:SalesforceData360Dialect"
123+
120124
[project.urls]
121125
"Source Code" = "https://github.com/DataKitchen/dataops-testgen"
122126
"Bug Tracker" = "https://github.com/DataKitchen/dataops-testgen/issues"
@@ -397,3 +401,7 @@ asset_dir = "ui/components/frontend/js"
397401
[[tool.streamlit.component.components]]
398402
name = "sidebar"
399403
asset_dir = "ui/components/frontend/js"
404+
405+
[[tool.streamlit.component.components]]
406+
name = "feedback_widget"
407+
asset_dir = "ui/components/frontend/js"

testgen/__main__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -982,7 +982,7 @@ def init_ui():
982982
"run",
983983
app_file,
984984
"--browser.gatherUsageStats=false",
985-
f"--logger.level={'debug' if settings.IS_DEBUG else 'error'}",
985+
"--logger.level=error",
986986
"--client.showErrorDetails=none",
987987
"--client.toolbarMode=minimal",
988988
"--server.enableStaticServing=true",

testgen/api/__init__.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
from fastapi import APIRouter
2+
3+
from testgen.api.app import router as _app_router
4+
from testgen.api.jobs import router as _jobs_router
5+
from testgen.api.runs import router as _runs_router
6+
from testgen.api.test_definitions import router as _test_definitions_router
7+
8+
router = APIRouter(prefix="/api/v1")
9+
router.include_router(_app_router)
10+
router.include_router(_jobs_router)
11+
router.include_router(_runs_router)
12+
router.include_router(_test_definitions_router)

testgen/api/app.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
from testgen.api.deps import db_session
66
from testgen.common import version_service
77

8-
router = APIRouter(prefix="/api/v1", tags=["API"], dependencies=[Depends(db_session)])
8+
router = APIRouter(tags=["API"], dependencies=[Depends(db_session)])
99

1010

1111
@router.get("/health")

testgen/api/deps.py

Lines changed: 26 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,14 @@
44

55
from fastapi import Depends, HTTPException, Security, status
66
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
7+
from sqlalchemy import select
78

89
from testgen.common.auth import authorize_token, decode_jwt_token
910
from testgen.common.models import Session, _current_session_wrapper, get_current_session
11+
from testgen.common.models.job_execution import PUBLIC_JOB_KEYS, JobExecution
1012
from testgen.common.models.project_membership import ProjectMembership
13+
from testgen.common.models.table_group import TableGroup
14+
from testgen.common.models.test_suite import TestSuite
1115
from testgen.common.models.user import User
1216
from testgen.utils.plugins import PluginHook
1317

@@ -73,14 +77,25 @@ def has_project_permission(user: User, project_code: str, permission: str) -> bo
7377

7478
# --- Resolver dependency factories ---
7579
# Each factory takes a permission string and returns Depends(). The entity ID
76-
# comes from a URL path parameter (FastAPI resolves it natively).
77-
# Entity not found and insufficient permission both raise the same 404
78-
# with a stable code/message — no variation that could leak the cause.
80+
# comes from a URL path parameter (FastAPI resolves it natively, including
81+
# UUID validation that yields a 422 for malformed inputs).
7982

8083
_require_user = Depends(get_authorized_user)
8184
_not_found = api_error(404, "not_found", "Not found")
8285

8386

87+
def _check_access(entity, user: User, permission: str):
88+
"""Return ``entity`` if the user has ``permission`` on its project, else raise 404.
89+
90+
Entity-not-found and insufficient-permission both surface as the same 404
91+
with a stable code/message — no variation that could leak the cause to an
92+
unauthorized caller.
93+
"""
94+
if entity and has_project_permission(user, entity.project_code, permission):
95+
return entity
96+
raise _not_found
97+
98+
8499
def resolve_project_code(permission: str):
85100
"""Verify the user has ``permission`` on the project identified by ``project_code`` path param."""
86101
def dependency(project_code: str, user: User = _require_user) -> str:
@@ -92,45 +107,31 @@ def dependency(project_code: str, user: User = _require_user) -> str:
92107

93108
def resolve_table_group(permission: str):
94109
"""Resolve a TableGroup by ``table_group_id`` path param and verify project permission."""
95-
from testgen.common.models.table_group import TableGroup
96-
97110
def dependency(table_group_id: UUID, user: User = _require_user) -> TableGroup:
98-
if (table_group := TableGroup.get(table_group_id)) and has_project_permission(user, table_group.project_code, permission):
99-
return table_group
100-
raise _not_found
111+
return _check_access(TableGroup.get(table_group_id), user, permission)
101112
return Depends(dependency)
102113

103114

104115
def resolve_test_suite(permission: str):
105116
"""Resolve a non-monitor TestSuite by ``test_suite_id`` path param and verify project permission."""
106-
from testgen.common.models.test_suite import TestSuite
107-
108117
def dependency(test_suite_id: UUID, user: User = _require_user) -> TestSuite:
109-
if (test_suite := TestSuite.get_regular(test_suite_id)) and has_project_permission(user, test_suite.project_code, permission):
110-
return test_suite
111-
raise _not_found
118+
return _check_access(TestSuite.get_regular(test_suite_id), user, permission)
112119
return Depends(dependency)
113120

114121

115122
def resolve_job(permission: str, *extra_filters):
116123
"""Resolve a JobExecution by ``job_id`` path param and verify project permission.
117124
118-
Internally-submitted jobs (source='system') are never exposed via the API.
119-
Extra ORM clauses are appended to the WHERE clause, e.g. to restrict by job_key.
120-
Mismatches surface as the same 404 — no information leakage.
125+
Only jobs whose ``job_key`` is in ``PUBLIC_JOB_KEYS`` are exposed via the API.
126+
Internal kinds (score rollups, recalculations, monitor runs) are filtered out
127+
by construction. Extra ORM clauses are appended to the WHERE clause to further
128+
restrict by job_key when a caller wants a single kind.
121129
"""
122-
from sqlalchemy import select
123-
124-
from testgen.common.models.job_execution import JobExecution
125-
126130
def dependency(job_id: UUID, user: User = _require_user) -> JobExecution:
127131
query = select(JobExecution).where(
128132
JobExecution.id == job_id,
129-
JobExecution.source != "system",
133+
JobExecution.job_key.in_(PUBLIC_JOB_KEYS),
130134
*extra_filters,
131135
)
132-
job = get_current_session().scalars(query).first()
133-
if job and has_project_permission(user, job.project_code, permission):
134-
return job
135-
raise _not_found
136+
return _check_access(get_current_session().scalars(query).first(), user, permission)
136137
return Depends(dependency)

testgen/api/jobs.py

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,17 @@
1010
resolve_table_group,
1111
resolve_test_suite,
1212
)
13-
from testgen.api.schemas import ErrorResponse, JobKey, JobListResponse, JobResponse, JobSource, JobSubmittedResponse
14-
from testgen.common.models.job_execution import JobExecution, JobStatus
13+
from testgen.api.schemas import ErrorResponse, JobListResponse, JobResponse, JobSubmittedResponse
14+
from testgen.common.enums import JobKey, JobSource, JobStatus
15+
from testgen.common.models.job_execution import PUBLIC_JOB_KEYS, JobExecution
1516
from testgen.common.models.table_group import TableGroup
1617
from testgen.common.models.test_suite import TestSuite
1718

1819
_error_responses = {
1920
404: {"model": ErrorResponse, "description": "Not found"},
2021
}
2122

22-
router = APIRouter(prefix="/api/v1", tags=["Jobs"], dependencies=[Depends(db_session)], responses=_error_responses)
23+
router = APIRouter(tags=["Jobs"], dependencies=[Depends(db_session)], responses=_error_responses)
2324

2425

2526
@router.post(
@@ -105,7 +106,7 @@ def list_jobs(
105106
"""List job executions for a project, with optional filters and pagination."""
106107
items, total = JobExecution.list_for_project(
107108
project_code,
108-
JobExecution.source != "system",
109+
JobExecution.job_key.in_(PUBLIC_JOB_KEYS),
109110
job_key=job_key,
110111
status=status,
111112
page=page,

testgen/api/runs.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
404: {"model": ErrorResponse, "description": "Not found"},
2727
}
2828

29-
router = APIRouter(prefix="/api/v1", tags=["runs"], dependencies=[Depends(db_session)], responses=_error_responses)
29+
router = APIRouter(tags=["runs"], dependencies=[Depends(db_session)], responses=_error_responses)
3030

3131

3232
@router.get(

testgen/api/schemas.py

Lines changed: 1 addition & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -6,27 +6,11 @@
66

77
from pydantic import BaseModel, field_validator
88

9-
from testgen.common.models.job_execution import JobStatus
9+
from testgen.common.enums import JobKey, JobSource, JobStatus
1010

1111
# --- Jobs ---
1212

1313

14-
class JobKey(StrEnum):
15-
run_profile = "run-profile"
16-
run_tests = "run-tests"
17-
run_monitors = "run-monitors"
18-
run_test_generation = "run-test-generation"
19-
20-
21-
class JobSource(StrEnum):
22-
api = "api"
23-
ui = "ui"
24-
scheduler = "scheduler"
25-
mcp = "mcp"
26-
cli = "cli"
27-
backfill = "backfill"
28-
29-
3014
class JobSubmittedResponse(BaseModel):
3115
"""Returned on 202 Accepted after successful job submission."""
3216

0 commit comments

Comments
 (0)