Merge pull request #69 from DataKitchen/release/5.70.2

Release/5.70.2

Author: datakitchen-devops

.gitignore

@@ -7,6 +7,8 @@ __pycache__/
 work_area/
 .envrc
 environment.list
+.claude/settings.local.json
+.claude/repo.yml
 
 # C extensions
 *.so

deploy/testgen-base.dockerfile

@@ -27,9 +27,7 @@ RUN apk update && apk upgrade && apk add --no-cache \
     openblas=0.3.30-r2 \
     openblas-dev=0.3.30-r2 \
     unixodbc=2.3.14-r0 \
-    unixodbc-dev=2.3.14-r0 \
-    libarrow=21.0.0-r4 \
-    apache-arrow-dev=21.0.0-r4
+    unixodbc-dev=2.3.14-r0
 
 COPY --chmod=775 ./deploy/install_linuxodbc.sh /tmp/dk/install_linuxodbc.sh
 RUN /tmp/dk/install_linuxodbc.sh
@@ -39,7 +37,7 @@ COPY ./pyproject.toml /tmp/dk/pyproject.toml
 RUN mkdir /dk
 
 # Upgrading pip for security
-RUN python3 -m pip install --no-cache-dir --upgrade pip==26.0
+RUN python3 -m pip install --no-cache-dir --upgrade pip==26.1.2
 
 # hdbcli only ships manylinux wheels (no musl). pip 26+ correctly rejects these on Alpine.
 # We download the wheel for the correct arch, then extract it directly into site-packages
@@ -73,8 +71,7 @@ RUN apk del \
     openssl \
     linux-headers \
     openblas-dev \
-    unixodbc-dev \
-    apache-arrow-dev
+    unixodbc-dev
 
 # Remove interactive ODBC tools — not needed at runtime, and iusql triggers
 # false-positive secret detection in security scanners (SECRET-3010)

deploy/testgen.dockerfile

@@ -1,4 +1,4 @@
-ARG TESTGEN_BASE_LABEL=v16
+ARG TESTGEN_BASE_LABEL=v17
 
 FROM datakitchen/dataops-testgen-base:${TESTGEN_BASE_LABEL} AS release-image
 

pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dataops-testgen"
-version = "5.48.0"
+version = "5.70.2"
 description = "DataKitchen's Data Quality DataOps TestGen"
 authors = [
     { "name" = "DataKitchen, Inc.", "email" = "info@datakitchen.io" },
@@ -61,7 +61,7 @@ dependencies = [
     "xlsxwriter==3.2.0",
     "psutil==5.9.8",
     "concurrent_log_handler==0.9.25",
-    "cryptography==46.0.6",
+    "cryptography==48.0.1",
     "validators==0.33.0",
     "reportlab==4.2.2",
     "cron-converter==1.2.1",
@@ -72,7 +72,7 @@ dependencies = [
     "holidays~=0.89",
 
     # Pinned to match the manually compiled libs or for security
-    "pyarrow==21.0.0",
+    "pyarrow==23.0.1",
     "matplotlib==3.9.2",
     "scipy==1.14.1",
     "jinja2==3.1.6",
@@ -82,7 +82,7 @@ dependencies = [
     # MCP server
     "mcp[cli]==1.26.0",
     "uvicorn==0.41.0",
-    "PyJWT==2.12.0",
+    "PyJWT==2.13.0",
     "bcrypt==5.0.0",
 
     # API & OAuth server
@@ -145,7 +145,7 @@ include-package-data = true
 include = [
       "testgen*",
 ]
-exclude = [ "*.tests", "tests*", "deploy*", "invocations*", "testlib*"]
+exclude = [ "*.tests", "tests*", "deploy*", "invocations*", "testgen.testing*"]
 
 [tool.pytest.ini_options]
 minversion = "7.0"

testgen/api/deps.py

@@ -6,9 +6,10 @@
 from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
 from sqlalchemy import select
 
-from testgen.common.auth import authorize_token, decode_jwt_token
+from testgen.common.auth import AuthError, authorize_token, decode_jwt_token
+from testgen.common.enums import PublicJobKey
 from testgen.common.models import Session, _current_session_wrapper, get_current_session
-from testgen.common.models.job_execution import PUBLIC_JOB_KEYS, JobExecution
+from testgen.common.models.job_execution import JobExecution
 from testgen.common.models.project_membership import ProjectMembership
 from testgen.common.models.table_group import TableGroup
 from testgen.common.models.test_suite import TestSuite
@@ -47,7 +48,7 @@ def get_authorized_user(credentials: HTTPAuthorizationCredentials = _bearer_secu
 
     try:
         payload = decode_jwt_token(credentials.credentials)
-    except ValueError:
+    except AuthError:
         raise _invalid from None
 
     username = payload.get("username")
@@ -57,7 +58,7 @@ def get_authorized_user(credentials: HTTPAuthorizationCredentials = _bearer_secu
     session = get_current_session()
     try:
         return authorize_token(credentials.credentials, username, session)
-    except ValueError:
+    except AuthError:
         raise _invalid from None
 
 
@@ -122,15 +123,15 @@ def dependency(test_suite_id: UUID, user: User = _require_user) -> TestSuite:
 def resolve_job(permission: str, *extra_filters):
     """Resolve a JobExecution by ``job_id`` path param and verify project permission.
 
-    Only jobs whose ``job_key`` is in ``PUBLIC_JOB_KEYS`` are exposed via the API.
+    Only externally-triggerable jobs (``PublicJobKey``) are exposed via the API.
     Internal kinds (score rollups, recalculations, monitor runs) are filtered out
     by construction. Extra ORM clauses are appended to the WHERE clause to further
     restrict by job_key when a caller wants a single kind.
     """
     def dependency(job_id: UUID, user: User = _require_user) -> JobExecution:
         query = select(JobExecution).where(
             JobExecution.id == job_id,
-            JobExecution.job_key.in_(PUBLIC_JOB_KEYS),
+            JobExecution.job_key.in_(list(PublicJobKey)),
             *extra_filters,
         )
         return _check_access(get_current_session().scalars(query).first(), user, permission)

testgen/api/enums.py

@@ -0,0 +1,52 @@
+"""Lowercase presentation enums for API v1 request filters and responses.
+
+Shared home for StrEnums that map a DB-stored value to the lowercase snake_case
+form exposed through the API. Several DB columns store title-case values
+(``test_results.result_status``, ``*.disposition``); the mapping dicts here are the
+single seam that normalizes them to the API surface — the DB is never changed.
+"""
+
+from enum import StrEnum
+
+from testgen.common.enums import Disposition as DbDisposition
+from testgen.common.models.test_result import TestResultStatus
+
+
+class ResultStatus(StrEnum):
+    """Outcome of a single test result."""
+
+    passed = "passed"
+    failed = "failed"
+    warning = "warning"
+    error = "error"
+    log = "log"
+
+
+class Disposition(StrEnum):
+    """Triage state of a test result. ``no_decision`` is the state of a result no one
+    has triaged yet. Omitting the filter returns active results (``confirmed`` and
+    ``no_decision``); pass an explicit value to filter to a single state."""
+
+    confirmed = "confirmed"
+    dismissed = "dismissed"
+    muted = "muted"
+    no_decision = "no_decision"
+
+
+RESULT_STATUS_TO_DB: dict[ResultStatus, TestResultStatus] = {
+    ResultStatus.passed: TestResultStatus.Passed,
+    ResultStatus.failed: TestResultStatus.Failed,
+    ResultStatus.warning: TestResultStatus.Warning,
+    ResultStatus.error: TestResultStatus.Error,
+    ResultStatus.log: TestResultStatus.Log,
+}
+RESULT_STATUS_FROM_DB: dict[TestResultStatus, ResultStatus] = {v: k for k, v in RESULT_STATUS_TO_DB.items()}
+
+# ``no_decision`` has no stored DB value — it corresponds to a NULL ``disposition``
+# column, handled explicitly at the API boundary (not present in these dicts).
+DISPOSITION_TO_DB: dict[Disposition, DbDisposition] = {
+    Disposition.confirmed: DbDisposition.CONFIRMED,
+    Disposition.dismissed: DbDisposition.DISMISSED,
+    Disposition.muted: DbDisposition.INACTIVE,
+}
+DISPOSITION_FROM_DB: dict[DbDisposition, Disposition] = {v: k for k, v in DISPOSITION_TO_DB.items()}

testgen/api/jobs.py

@@ -11,8 +11,8 @@
     resolve_test_suite,
 )
 from testgen.api.schemas import ErrorResponse, JobListResponse, JobResponse, JobSubmittedResponse
-from testgen.common.enums import JobKey, JobSource, JobStatus
-from testgen.common.models.job_execution import PUBLIC_JOB_KEYS, JobExecution
+from testgen.common.enums import JobKey, JobSource, JobStatus, PublicJobKey
+from testgen.common.models.job_execution import JobExecution
 from testgen.common.models.table_group import TableGroup
 from testgen.common.models.test_suite import TestSuite
 
@@ -98,15 +98,15 @@ def cancel_job(job: JobExecution = resolve_job("edit")):  # noqa: B008
 )
 def list_jobs(
     project_code: str = resolve_project_code("view"),
-    job_key: JobKey | None = Query(default=None),  # noqa: B008
+    job_key: PublicJobKey | None = Query(default=None),  # noqa: B008
     status: JobStatus | None = Query(default=None),  # noqa: B008
     page: int = Query(default=1, ge=1),
     limit: int = Query(default=20, ge=1, le=100),
 ):
     """List job executions for a project, with optional filters and pagination."""
     items, total = JobExecution.list_for_project(
         project_code,
-        JobExecution.job_key.in_(PUBLIC_JOB_KEYS),
+        JobExecution.job_key.in_(list(PublicJobKey)),
         job_key=job_key,
         status=status,
         page=page,

testgen/api/oauth/metadata.py

@@ -12,7 +12,7 @@
 def authorization_server_metadata():
     """Return OAuth 2.1 Authorization Server Metadata per RFC 8414.
 
-    MCP clients use this for server discovery.
+    OAuth clients use this for server discovery.
     """
     base_url = settings.BASE_URL.rstrip("/")
 

testgen/api/oauth/models.py

@@ -21,10 +21,10 @@
 from testgen import settings
 from testgen.api.deps import db_session
 from testgen.api.oauth.login import render_login_page
-from testgen.api.oauth.models import OAuth2Client
 from testgen.api.oauth.server import TestGenAuthorizationServer
 from testgen.common.auth import create_jwt_token, decode_jwt_token, verify_password
 from testgen.common.models import get_current_session
+from testgen.common.models.oauth import OAuth2Client
 from testgen.common.models.user import User
 
 LOG = logging.getLogger("testgen")