Merge pull request #65 from DataKitchen/release/5.32.2

Release/5.32.2

Author: datakitchen-devops

.github/actions/publish_charts/action.yaml

@@ -22,7 +22,7 @@ runs:
         helm repo add bitnami https://charts.bitnami.com/bitnami
 
     - name: Run chart-releaser
-      uses: helm/chart-releaser-action@v1.6.0
+      uses: helm/chart-releaser-action@v1.7.0
       with:
         charts_dir: deploy/charts
         skip_existing: 'true'

deploy/build_api_docs.py

@@ -0,0 +1,50 @@
+"""Export the TestGen OpenAPI spec as a JSON file.
+
+Usage:
+    python deploy/build_api_docs.py [--output PATH] [--version VERSION]
+
+The output JSON is served by a static Redoc HTML shell alongside it.
+"""
+
+import argparse
+import json
+from pathlib import Path
+
+from testgen.server import create_app
+
+_REPO_ROOT = Path(__file__).resolve().parent.parent
+
+
+def _read_version_from_pyproject() -> str:
+    try:
+        import tomllib  # Python 3.11+
+    except ImportError:
+        import tomli as tomllib  # type: ignore[no-redef]
+
+    with open(_REPO_ROOT / "pyproject.toml", "rb") as f:
+        return tomllib.load(f)["project"]["version"]
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Export the TestGen OpenAPI spec as JSON.")
+    parser.add_argument(
+        "--output",
+        type=Path,
+        default=Path("docs/api/openapi.json"),
+        help="Output JSON file path (default: docs/api/openapi.json, relative to cwd)",
+    )
+    parser.add_argument("--version", help="API version string (default: read from pyproject.toml)")
+    args = parser.parse_args()
+
+    version = args.version or _read_version_from_pyproject()
+    app = create_app(version=version)
+    spec = app.openapi()
+
+    output: Path = args.output
+    output.parent.mkdir(parents=True, exist_ok=True)
+    output.write_text(json.dumps(spec, indent=2) + "\n", encoding="utf-8")
+    print(f"Exported OpenAPI spec -> {output} (v{version})")
+
+
+if __name__ == "__main__":
+    main()

deploy/build_mcp_docs.py

@@ -0,0 +1,151 @@
+"""Export the TestGen MCP server as a Markdown reference page.
+
+Usage:
+    python deploy/build_mcp_docs.py [--output PATH]
+
+Introspects the FastMCP instance built by ``build_mcp_server()`` and emits
+a single Markdown page listing prompts, tools, and resources. Tools are
+grouped by the ``_DOC_GROUP`` constant defined on each tool module — when
+adding a new tool module, declare ``_DOC_GROUP = "..."`` so the new tools
+land under the right heading automatically.
+"""
+
+import argparse
+import re
+import sys
+import textwrap
+from pathlib import Path
+from typing import Any
+
+from testgen.mcp.server import build_mcp_server
+from testgen.mcp.tools.common import DocGroup
+
+_DEFAULT_OUTPUT = Path("docs/mcp/supported-tools.md")
+_ARGS_HEADER_RE = re.compile(r"^\s*Args:\s*$", re.MULTILINE)
+
+# Order in which tool groups appear on the page. Each entry is a ``DocGroup``
+# member; tools whose module declares a ``_DOC_GROUP`` not in this list are
+# appended after these in the order they are first seen.
+_GROUP_ORDER: list[DocGroup] = [
+    DocGroup.DISCOVER,
+    DocGroup.INVESTIGATE,
+    DocGroup.BROWSE_PROFILING,
+    DocGroup.TRIGGER,
+]
+_FALLBACK_GROUP = "Other tools"
+
+
+def _short_description(docstring: str) -> str:
+    """Return the first prose paragraph of a docstring, stripped of Args/Returns sections."""
+    if not docstring:
+        return ""
+    text = textwrap.dedent(docstring).strip()
+    match = _ARGS_HEADER_RE.search(text)
+    if match:
+        text = text[: match.start()].rstrip()
+    first_paragraph = text.split("\n\n", 1)[0]
+    return " ".join(line.strip() for line in first_paragraph.splitlines())
+
+
+def _entry_name(item: Any) -> str:
+    """Display name for a tool, resource, or prompt."""
+    return str(getattr(item, "uri", None) or item.name)
+
+
+def _render_entry(item: Any) -> str:
+    description = _short_description(item.description or "")
+    return f"- **`{_entry_name(item)}`** — {description}"
+
+
+def _group_for_tool(tool: Any) -> str:
+    """Resolve a tool's display group via its module's ``_DOC_GROUP`` constant."""
+    module = sys.modules.get(tool.fn.__module__)
+    group = getattr(module, "_DOC_GROUP", None)
+    return str(group) if group is not None else _FALLBACK_GROUP
+
+
+def _group_tools(tools: list[Any]) -> list[tuple[str, list[Any]]]:
+    """Bucket tools by their module's ``_DOC_GROUP``, ordered by ``_GROUP_ORDER``."""
+    buckets: dict[str, list[Any]] = {}
+    for tool in tools:
+        buckets.setdefault(_group_for_tool(tool), []).append(tool)
+
+    ordered: list[tuple[str, list[Any]]] = []
+    for group in _GROUP_ORDER:
+        title = str(group)
+        if title in buckets:
+            ordered.append((title, sorted(buckets.pop(title), key=lambda t: t.name)))
+    for title, bucket in buckets.items():
+        ordered.append((title, sorted(bucket, key=lambda t: t.name)))
+    return ordered
+
+
+def _build_markdown(mcp: Any) -> str:
+    tools = mcp._tool_manager.list_tools()
+    resources = sorted(mcp._resource_manager.list_resources(), key=lambda r: str(r.uri))
+    prompts = sorted(mcp._prompt_manager.list_prompts(), key=lambda p: p.name)
+    grouped_tools = _group_tools(list(tools))
+
+    parts: list[str] = [
+        "# Supported Tools",
+        "",
+        "The TestGen MCP server exposes the prompts, tools, and resources listed below.",
+        "",
+        "For setup instructions, see [Set up the MCP Server](setup.md).",
+        "For example questions to ask an assistant, see [MCP Server](index.md#what-you-can-ask).",
+        "",
+        "## Prompts",
+        "",
+        (
+            "Prompts are pre-built workflows you can invoke directly through your AI client — typically "
+            "as a slash command (for example, `/testgen:table_health` in Claude Code) or "
+            "from a quick-action menu. They orchestrate several tool calls behind the scenes for common "
+            "investigations. Exact UX varies by client."
+        ),
+        "",
+    ]
+    parts.extend(_render_entry(prompt) for prompt in prompts)
+    parts.append("")
+
+    parts.extend(["## Tools", "", "Tools are operations the assistant calls during a conversation, picked based on what you ask.", ""])
+    for heading, bucket in grouped_tools:
+        parts.append(f"### {heading}")
+        parts.append("")
+        parts.extend(_render_entry(tool) for tool in bucket)
+        parts.append("")
+
+    parts.extend(
+        [
+            "## Resources",
+            "",
+            "Resources are static reference documents that AI clients can fetch by URI.",
+            "",
+        ]
+    )
+    parts.extend(_render_entry(resource) for resource in resources)
+
+    return "\n".join(parts).rstrip() + "\n"
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Export the TestGen MCP server as a Markdown reference.")
+    parser.add_argument(
+        "--output",
+        type=Path,
+        default=_DEFAULT_OUTPUT,
+        help=f"Output Markdown file path (default: {_DEFAULT_OUTPUT}, relative to cwd)",
+    )
+    args = parser.parse_args()
+
+    mcp = build_mcp_server(api_base_url="https://testgen.example.com")
+    markdown = _build_markdown(mcp)
+
+    output: Path = args.output
+    output.parent.mkdir(parents=True, exist_ok=True)
+    frontmatter = "---\nsearch:\n  boost: 0.5\n---\n"
+    output.write_text(frontmatter + markdown, encoding="utf-8")
+    print(f"Exported MCP supported tools -> {output}")
+
+
+if __name__ == "__main__":
+    main()

deploy/charts/testgen-app/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.1.0
+version: 1.2.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

deploy/charts/testgen-app/templates/_environment.yaml

@@ -43,6 +43,10 @@
 - name: TG_EMAIL_FROM_ADDRESS
   value: {{ .fromAddress | quote }}
 {{- end -}}
+{{- if .Values.testgen.uiBaseUrl }}
+- name: TG_UI_BASE_URL
+  value: {{ .Values.testgen.uiBaseUrl | quote }}
+{{- end }}
 {{- end -}}
 
 {{- define "testgen.hookEnvironment" -}}

deploy/charts/testgen-app/values.yaml

@@ -22,6 +22,7 @@ testgen:
       port:
       username:
       password:
+  uiBaseUrl:
   labels:
 
 cliHooks:

deploy/testgen-base.dockerfile

@@ -29,9 +29,7 @@ RUN apk update && apk upgrade && apk add --no-cache \
     unixodbc=2.3.14-r0 \
     unixodbc-dev=2.3.14-r0 \
     libarrow=21.0.0-r4 \
-    apache-arrow-dev=21.0.0-r4 \
-    # Pinned versions for security
-    xz=5.8.2-r0
+    apache-arrow-dev=21.0.0-r4
 
 COPY --chmod=775 ./deploy/install_linuxodbc.sh /tmp/dk/install_linuxodbc.sh
 RUN /tmp/dk/install_linuxodbc.sh

deploy/testgen.dockerfile

@@ -1,4 +1,4 @@
-ARG TESTGEN_BASE_LABEL=v14
+ARG TESTGEN_BASE_LABEL=v15
 
 FROM datakitchen/dataops-testgen-base:${TESTGEN_BASE_LABEL} AS release-image
 

docs/configuration.md

@@ -282,3 +282,15 @@ default: `dataset`
 
 When exporting to your instance of Observabilty, the key sent to the events API to identify the components.
 default: `default`
+
+### URL Configuration
+
+#### `TG_UI_BASE_URL`
+
+Externally-reachable base URL for the TestGen web UI. Used in email notification links and PDF report links so recipients can click through to the correct address.
+
+Must be set in production when TestGen is behind a reverse proxy or load balancer. If not set, defaults to `http://localhost:<STREAMLIT_SERVER_PORT>`.
+
+Example: `https://testgen.example.com`
+
+default: `http://localhost:8501`

invocations/dev.py

@@ -1,4 +1,4 @@
-__all__ = ["build_public_image", "clean", "install", "lint"]
+__all__ = ["build_api_docs", "build_mcp_docs", "build_public_image", "clean", "install", "lint"]
 
 import re
 from os.path import exists, join
@@ -72,6 +72,26 @@ def clean(ctx: Context) -> None:
     print("Cleaning finished!")
 
 
+@task(name="build-api-docs", pre=(install,))
+def build_api_docs(ctx: Context, version: str = "", output: str = "") -> None:
+    """Exports the OpenAPI spec as JSON for the static API docs."""
+    args = []
+    if version:
+        args.append(f"--version {version}")
+    if output:
+        args.append(f"--output {output}")
+    ctx.run(f"python deploy/build_api_docs.py {' '.join(args)}")
+
+
+@task(name="build-mcp-docs", pre=(install,))
+def build_mcp_docs(ctx: Context, output: str = "") -> None:
+    """Exports the MCP supported-tools page from the FastMCP server."""
+    args = []
+    if output:
+        args.append(f"--output {output}")
+    ctx.run(f"python deploy/build_mcp_docs.py {' '.join(args)}")
+
+
 @task(
     pre=(required_tools, prep_dk_builer),
     iterable=["label"],