✨Feat:Integrate openjiuwen's intelligent agent evaluation function close#3188 (#3356)

* ✨Feat：add prompt optimization

* 🐛Bugfix: dockerbuild failed when running pipefail in python3_11

* 🔨Optimize： Optimize prompt optimization display page and interaction methods

* ✨Feat:add agent evaluator

* 🐛Bugfix: fix fronted and report download

* 🐛 Bugfix: Add xlrd dependency and improve openjiuwen SDK adapter to handle circular imports more effectively. Update import paths in frontend components for consistency.

* ♻️Refactor: frontend style refact

* 🔨Rename evaluator sql

* ♻️ Refactor: Optimize the frontend page for agent evaluation

* ♻️Refactor: update modal mask properties for consistency

* ♻️ Refactor: Simplify locale handling and clean up imports in OAuthCompletePage and UserManageComp

* ✨ Feature: Enhance agent evaluation with Chinese output directive and new database migrations

* 🔧 Update: Change button variant from "outline" to "outlined" in EvaluationConfigCard and EvaluationReportCard components; enhance datetime handling in test_client.py to ensure correct timezone representation.

* 🧪Test: Add unit tests for the jiuwen_sdk_adapter to verify module-level behavior and side effects.

* 🔧 Update: Refactor the mock invocation in test_jiuwen_sdk_adapter to use an async function for better compatibility with asynchronous tests.

* 🧪 Test: Add comprehensive unit tests for evaluation set functionality, including CRUD operations and Excel parsing, to ensure robust behavior and error handling.

* 🧪 Test: Add extensive unit tests for Jiuwen SDK adapter and evaluation services, covering lazy imports, log envelope parsing, and case validation to enhance error handling and ensure robust functionality.

* 🔧 Update: Consolidate judge_model_id addition into agent_evaluation_t migration, removing the separate migration file for improved installation simplicity and ensuring idempotency in existing environments.

* 🔧 Update: Introduce a consolidated migration file for agent evaluation, combining previous separate migrations into a single idempotent script for improved installation simplicity and clarity. This includes the addition of evaluation tables and the pass_status column, ensuring all necessary structures are created in one step.

Author: DongJiBao2001

backend/adapters/jiuwen_sdk_adapter.py

@@ -0,0 +1,147 @@
+import logging
+from http import HTTPStatus
+from typing import Optional
+
+from fastapi import APIRouter, Body, Header, HTTPException, Query
+from fastapi.responses import JSONResponse, StreamingResponse
+
+from services.agent_evaluation_service import (
+    create_agent_evaluation_run_impl,
+    delete_agent_evaluation_run_impl,
+    generate_agent_evaluation_report_impl,
+    get_agent_evaluation_run_impl,
+    list_agent_evaluation_cases_impl,
+    list_agent_evaluations_by_agent_impl,
+)
+from utils.auth_utils import get_current_user_id
+
+logger = logging.getLogger("agent_evaluation_app")
+
+router = APIRouter(prefix="/agent-evaluations")
+
+
+@router.post("")
+async def create_agent_evaluation_api(
+    agent_id: int = Body(...),
+    evaluation_set_id: int = Body(...),
+    judge_model_id: int = Body(..., description="Model id used for judging (Jiuwen)"),
+    authorization: Optional[str] = Header(None),
+):
+    try:
+        user_id, tenant_id = get_current_user_id(authorization)
+        run = create_agent_evaluation_run_impl(
+            tenant_id=tenant_id,
+            user_id=user_id,
+            agent_id=agent_id,
+            evaluation_set_id=evaluation_set_id,
+            judge_model_id=judge_model_id,
+        )
+        return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": run})
+    except ValueError as ve:
+        raise HTTPException(status_code=400, detail=str(ve))
+    except Exception as exc:
+        logger.exception("Create agent evaluation error: %r", exc)
+        raise HTTPException(status_code=500, detail="Create agent evaluation error")
+
+
+@router.get("")
+async def list_agent_evaluations_by_agent_api(
+    agent_id: int = Query(...),
+    limit: int = Query(50, ge=1, le=200),
+    offset: int = Query(0, ge=0),
+    authorization: Optional[str] = Header(None),
+):
+    try:
+        _, tenant_id = get_current_user_id(authorization)
+        data = list_agent_evaluations_by_agent_impl(
+            agent_id=agent_id,
+            tenant_id=tenant_id,
+            limit=limit,
+            offset=offset,
+        )
+        return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": data})
+    except Exception as exc:
+        logger.exception("List agent evaluations error: %r", exc)
+        raise HTTPException(status_code=500, detail="List agent evaluations error")
+
+
+@router.get("/{agent_evaluation_id}")
+async def get_agent_evaluation_api(
+    agent_evaluation_id: int,
+    authorization: Optional[str] = Header(None),
+):
+    try:
+        _, tenant_id = get_current_user_id(authorization)
+        data = get_agent_evaluation_run_impl(agent_evaluation_id=agent_evaluation_id, tenant_id=tenant_id)
+        return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": data})
+    except Exception as exc:
+        logger.exception("Get agent evaluation error: %r", exc)
+        raise HTTPException(status_code=500, detail="Get agent evaluation error")
+
+
+@router.get("/{agent_evaluation_id}/cases")
+async def list_agent_evaluation_cases_api(
+    agent_evaluation_id: int,
+    limit: int = Query(50, ge=1, le=200),
+    offset: int = Query(0, ge=0),
+    authorization: Optional[str] = Header(None),
+):
+    try:
+        _, tenant_id = get_current_user_id(authorization)
+        data = list_agent_evaluation_cases_impl(
+            agent_evaluation_id=agent_evaluation_id,
+            tenant_id=tenant_id,
+            limit=limit,
+            offset=offset,
+        )
+        return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": data})
+    except Exception as exc:
+        logger.exception("List agent evaluation cases error: %r", exc)
+        raise HTTPException(status_code=500, detail="List agent evaluation cases error")
+
+
+@router.get("/{agent_evaluation_id}/report")
+async def download_agent_evaluation_report_api(
+    agent_evaluation_id: int,
+    authorization: Optional[str] = Header(None),
+):
+    try:
+        _, tenant_id = get_current_user_id(authorization)
+        data, fail_count = generate_agent_evaluation_report_impl(
+            agent_evaluation_id=agent_evaluation_id,
+            tenant_id=tenant_id,
+        )
+        suffix = "_failed.xlsx" if fail_count > 0 else "_all.xlsx"
+        return StreamingResponse(
+            iter([data]),
+            media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+            headers={
+                "Content-Disposition": f"attachment; filename=evaluation_report_{agent_evaluation_id}{suffix}"
+            },
+        )
+    except ValueError as ve:
+        raise HTTPException(status_code=404, detail=str(ve))
+    except Exception as exc:
+        logger.exception("Download agent evaluation report error: %r", exc)
+        raise HTTPException(status_code=500, detail="Download agent evaluation report error")
+
+
+@router.delete("/{agent_evaluation_id}")
+async def delete_agent_evaluation_api(
+    agent_evaluation_id: int,
+    authorization: Optional[str] = Header(None),
+):
+    """Soft-delete an evaluation run. Only the creator may delete."""
+    try:
+        user_id, tenant_id = get_current_user_id(authorization)
+        delete_agent_evaluation_run_impl(
+            agent_evaluation_id=agent_evaluation_id,
+            tenant_id=tenant_id,
+            user_id=user_id,
+        )
+        return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success"})
+    except ValueError as ve:
+        raise HTTPException(status_code=400, detail=str(ve))
+    except Exception as exc:
+        logger.exception("Delete agent evaluation error: %r", exc)
+        raise HTTPException(status_code=500, detail="Delete agent evaluation error")

backend/apps/agent_evaluation_app.py

@@ -33,6 +33,8 @@
 from apps.monitoring_app import router as monitoring_router
 from apps.a2a_server_app import router as a2a_server_router
 from apps.haotian_app import router as haotian_router
+from apps.evaluation_set_app import router as evaluation_set_router
+from apps.agent_evaluation_app import router as agent_evaluation_router
 from apps.aidp_app import router as aidp_router
 from apps.cas_app import router as cas_router
 from consts.const import IS_SPEED_MODE
@@ -93,4 +95,6 @@ async def sync_default_prompt_template_on_startup():
 app.include_router(a2a_client_router)
 app.include_router(a2a_server_router)
 app.include_router(haotian_router)
+app.include_router(evaluation_set_router)
+app.include_router(agent_evaluation_router)
 app.include_router(aidp_router)

backend/apps/config_app.py

@@ -0,0 +1,198 @@
+import io
+import json
+import logging
+from http import HTTPStatus
+from typing import Any, Dict, List, Optional
+
+from fastapi import APIRouter, Body, File, Form, Header, HTTPException, Query, UploadFile
+from fastapi.responses import JSONResponse, StreamingResponse
+
+from services.evaluation_set_service import (
+    create_evaluation_set_from_cases,
+    create_evaluation_set_from_jsonl,
+    delete_evaluation_set_impl,
+    get_evaluation_set_impl,
+    list_evaluation_set_cases_impl,
+    list_evaluation_sets_impl,
+)
+from utils.auth_utils import get_current_user_id
+from utils.evaluation_set_excel_utils import build_evaluation_set_excel_template_bytes, parse_evaluation_cases_from_excel
+
+logger = logging.getLogger("evaluation_set_app")
+
+router = APIRouter(prefix="/evaluation-sets")
+
+
+@router.get("")
+async def list_evaluation_sets_api(
+    limit: int = Query(50, ge=1, le=200),
+    offset: int = Query(0, ge=0),
+    authorization: Optional[str] = Header(None),
+):
+    try:
+        user_id, tenant_id = get_current_user_id(authorization)
+        data = list_evaluation_sets_impl(tenant_id=tenant_id, limit=limit, offset=offset)
+        return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": data})
+    except Exception as exc:
+        logger.exception("List evaluation sets error: %r", exc)
+        raise HTTPException(status_code=500, detail="List evaluation sets error")
+
+
+@router.post("")
+async def create_evaluation_set_api(
+    name: str = Body(...),
+    description: Optional[str] = Body(None),
+    source_filename: Optional[str] = Body(None),
+    jsonl_text: str = Body(..., description="Raw JSONL content"),
+    authorization: Optional[str] = Header(None),
+):
+    try:
+        user_id, tenant_id = get_current_user_id(authorization)
+        meta = create_evaluation_set_from_jsonl(
+            tenant_id=tenant_id,
+            name=name,
+            description=description,
+            source_filename=source_filename,
+            jsonl_text=jsonl_text,
+            created_by=user_id,
+        )
+        return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": meta})
+    except ValueError as ve:
+        raise HTTPException(status_code=400, detail=str(ve))
+    except Exception as exc:
+        logger.exception("Create evaluation set error: %r", exc)
+        raise HTTPException(status_code=500, detail="Create evaluation set error")
+
+
+@router.post("/upload")
+async def upload_evaluation_set_api(
+    name: str = Form(...),
+    description: Optional[str] = Form(None),
+    files: List[UploadFile] = File(...),
+    authorization: Optional[str] = Header(None, alias="Authorization"),
+):
+    try:
+        user_id, tenant_id = get_current_user_id(authorization)
+        if not files:
+            raise ValueError("At least one file is required")
+
+        all_cases: List[Dict[str, Any]] = []
+        source_filenames: List[str] = []
+
+        for file in files:
+            raw = await file.read()
+            filename = file.filename or ""
+            source_filenames.append(filename)
+            lower = filename.lower()
+
+            if lower.endswith(".xlsx") or lower.endswith(".xls"):
+                cases = parse_evaluation_cases_from_excel(filename=filename, raw=raw)
+                all_cases.extend(cases)
+            else:
+                # Backward compatible: still accept JSONL upload
+                try:
+                    jsonl_text = raw.decode("utf-8")
+                except Exception:
+                    jsonl_text = raw.decode("utf-8", errors="ignore")
+
+                # Parse JSONL into cases
+                for line in jsonl_text.strip().splitlines():
+                    line = line.strip()
+                    if not line:
+                        continue
+                    obj = json.loads(line)
+                    all_cases.append({
+                        "query": obj.get("query", ""),
+                        "answer": obj.get("answer", ""),
+                        "context": obj.get("context"),
+                        "case_id": obj.get("case_id"),
+                    })
+
+        if not all_cases:
+            raise ValueError("No valid cases found in uploaded files")
+
+        meta = create_evaluation_set_from_cases(
+            tenant_id=tenant_id,
+            name=name,
+            description=description,
+            source_filename=", ".join(source_filenames),
+            cases=all_cases,
+            created_by=user_id,
+        )
+
+        return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": meta})
+    except ValueError as ve:
+        raise HTTPException(status_code=400, detail=str(ve))
+    except Exception as exc:
+        logger.exception("Upload evaluation set error: %r", exc)
+        raise HTTPException(status_code=500, detail="Upload evaluation set error")
+
+
+@router.get("/template")
+async def download_evaluation_set_template_api():
+    """Download Excel template for evaluation set upload."""
+    data = build_evaluation_set_excel_template_bytes()
+    headers = {
+        "Content-Disposition": 'attachment; filename="evaluation_set_template.xlsx"'
+    }
+    return StreamingResponse(
+        io.BytesIO(data),
+        media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        headers=headers,
+    )
+
+
+@router.get("/{evaluation_set_id}")
+async def get_evaluation_set_api(
+    evaluation_set_id: int,
+    authorization: Optional[str] = Header(None),
+):
+    try:
+        _, tenant_id = get_current_user_id(authorization)
+        data = get_evaluation_set_impl(evaluation_set_id=evaluation_set_id, tenant_id=tenant_id)
+        return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": data})
+    except Exception as exc:
+        logger.exception("Get evaluation set error: %r", exc)
+        raise HTTPException(status_code=500, detail="Get evaluation set error")
+
+
+@router.get("/{evaluation_set_id}/cases")
+async def list_evaluation_set_cases_api(
+    evaluation_set_id: int,
+    limit: int = Query(50, ge=1, le=200),
+    offset: int = Query(0, ge=0),
+    authorization: Optional[str] = Header(None),
+):
+    try:
+        _, tenant_id = get_current_user_id(authorization)
+        data = list_evaluation_set_cases_impl(
+            evaluation_set_id=evaluation_set_id,
+            tenant_id=tenant_id,
+            limit=limit,
+            offset=offset,
+        )
+        return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": data})
+    except Exception as exc:
+        logger.exception("List evaluation set cases error: %r", exc)
+        raise HTTPException(status_code=500, detail="List evaluation set cases error")
+
+
+@router.delete("/{evaluation_set_id}")
+async def delete_evaluation_set_api(
+    evaluation_set_id: int,
+    authorization: Optional[str] = Header(None),
+):
+    """Soft-delete an evaluation set.
+
+    Blocked when any active evaluation run still references the set, so
+    historical runs never lose their context.
+    """
+    try:
+        user_id, tenant_id = get_current_user_id(authorization)
+        delete_evaluation_set_impl(evaluation_set_id, tenant_id, user_id)
+        return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success"})
+    except ValueError as ve:
+        raise HTTPException(status_code=400, detail=str(ve))
+    except Exception as exc:
+        logger.exception("Delete evaluation set error: %r", exc)
+        raise HTTPException(status_code=500, detail="Delete evaluation set error")