diff --git a/backend/agents/create_agent_info.py b/backend/agents/create_agent_info.py index e4fc5ab46..646a44f61 100644 --- a/backend/agents/create_agent_info.py +++ b/backend/agents/create_agent_info.py @@ -6,7 +6,7 @@ from jinja2 import Template, StrictUndefined from nexent.core.utils.observer import MessageObserver -from nexent.core.agents.agent_model import AgentRunInfo, ModelConfig, AgentConfig, ToolConfig, ExternalA2AAgentConfig, AgentHistory +from nexent.core.agents.agent_model import AgentRunInfo, ModelConfig, AgentConfig, ToolConfig, ExternalA2AAgentConfig, AgentHistory, AgentVerificationConfig from nexent.core.agents.agent_context import ContextManagerConfig from nexent.memory.memory_service import search_memory_in_levels @@ -565,6 +565,7 @@ async def create_agent_config( external_a2a_agents=external_a2a_agents, context_manager_config=cm_config, context_components=context_components, + verification_config=AgentVerificationConfig.model_validate(agent_info.get("verification_config") or {}), ) return agent_config diff --git a/backend/consts/model.py b/backend/consts/model.py index 370599c82..357f49ada 100644 --- a/backend/consts/model.py +++ b/backend/consts/model.py @@ -2,7 +2,7 @@ from typing import Optional, Any, List, Dict, Literal from pydantic import BaseModel, Field, EmailStr, ConfigDict, field_validator -from nexent.core.agents.agent_model import ToolConfig +from nexent.core.agents.agent_model import AgentVerificationConfig, ToolConfig from consts.prompt_template import PROMPT_GENERATE_TEMPLATE_FIELD_ALIAS_MAP @@ -489,10 +489,18 @@ class AgentInfoRequest(BaseModel): group_ids: Optional[List[int]] = None ingroup_permission: Optional[str] = None enable_context_manager: Optional[bool] = None + verification_config: Optional[Dict[str, Any]] = None greeting_message: Optional[str] = None example_questions: Optional[List[str]] = None version_no: int = 0 + @field_validator("verification_config", mode="before") + @classmethod + def normalize_verification_config(cls, value): + if value is None: + return None + return AgentVerificationConfig.model_validate(value).model_dump() + class AgentIDRequest(BaseModel): agent_id: int @@ -565,6 +573,7 @@ class ExportAndImportAgentInfo(BaseModel): author: Optional[str] = None max_steps: int provide_run_summary: bool + verification_config: Optional[Dict[str, Any]] = None duty_prompt: Optional[str] = None constraint_prompt: Optional[str] = None few_shots_prompt: Optional[str] = None diff --git a/backend/database/agent_db.py b/backend/database/agent_db.py index 1cb589c11..533659b0f 100644 --- a/backend/database/agent_db.py +++ b/backend/database/agent_db.py @@ -198,6 +198,7 @@ def create_agent(agent_info, tenant_id: str, user_id: str): """ info_with_metadata = dict(agent_info) info_with_metadata.setdefault("max_steps", 15) + info_with_metadata.setdefault("verification_config", None) info_with_metadata.update({ "tenant_id": tenant_id, "version_no": 0, # Default to draft version @@ -236,6 +237,7 @@ def create_agent(agent_info, tenant_id: str, user_id: str): "group_ids": new_agent.group_ids, "is_new": new_agent.is_new, "enable_context_manager": new_agent.enable_context_manager, + "verification_config": new_agent.verification_config, "greeting_message": new_agent.greeting_message, "example_questions": new_agent.example_questions, "current_version_no": new_agent.current_version_no, diff --git a/backend/database/db_models.py b/backend/database/db_models.py index ef72df6b2..5450b5f74 100644 --- a/backend/database/db_models.py +++ b/backend/database/db_models.py @@ -333,6 +333,7 @@ class AgentInfo(TableBase): current_version_no = Column(Integer, nullable=True, doc="Current published version number. NULL means no version published yet") ingroup_permission = Column(String(30), doc="In-group permission: EDIT, READ_ONLY, PRIVATE") enable_context_manager = Column(Boolean, default=False, doc="Whether to enable context management (compression) for this agent") + verification_config = Column(JSONB, doc="Layered ReAct self-verification configuration") greeting_message = Column(Text, doc="Agent greeting message displayed on chat initial screen") example_questions = Column(JSONB, doc="List of example questions for starting a conversation with this agent") diff --git a/backend/prompts/managed_system_prompt_template_en.yaml b/backend/prompts/managed_system_prompt_template_en.yaml index dfc44fec2..62e16e946 100644 --- a/backend/prompts/managed_system_prompt_template_en.yaml +++ b/backend/prompts/managed_system_prompt_template_en.yaml @@ -66,6 +66,11 @@ system_prompt: |- - Note that executed code is not visible to users. If users need to see the code, use 'code' for displaying code. - **IMPORTANT**: After code execution, the system will return content with "Observation:" marker (this is the real execution result). Please continue your next thinking based on these real results. **Do NOT fabricate observation results before code execution.** + 3. Self-verification: + - After critical events (tool calls, retrieval results, code execution, and final-answer preparation), the system may run explicit verification. + - If verification reports errors, insufficient evidence, incomplete parameters, or unreliable results, you must repair the issue, gather more evidence, call tools again, or clearly state what cannot be completed. + - The final answer is shown to the user only after verification passes. If the system returns Verification feedback, treat it as a real observation and continue revising. + After thinking, when you believe you can answer the user's question, you can generate a final answer directly to the user without generating code and stop the loop. When generating the final answer, you need to follow these specifications: @@ -178,3 +183,13 @@ final_answer: Original task: {{task}} Please provide a clear and concise summary of the work completed so far. + + +verification: + pre_messages: |- + You are a strict verifier for a ReAct agent. Judge reliability only from the task, candidate answer, tool outputs, and observations. Do not output hidden chain-of-thought. + You must output JSON only. + + post_messages: |- + Verify whether the candidate answer covers the user's intent, is grounded in observations, handles tool errors, uses trustworthy citations, and is formatted for users. + Output fields: passed, score, status, failed_criteria, checks, revision_instruction, user_visible_note. diff --git a/backend/prompts/managed_system_prompt_template_zh.yaml b/backend/prompts/managed_system_prompt_template_zh.yaml index 750241b92..da3d53469 100644 --- a/backend/prompts/managed_system_prompt_template_zh.yaml +++ b/backend/prompts/managed_system_prompt_template_zh.yaml @@ -130,6 +130,11 @@ system_prompt: |- - 注意运行的代码不会被用户看到，所以如果用户需要看到代码，你需要使用'代码'表达展示代码。 - **重要**：代码执行后，系统会返回 "Observation:" 标记的内容（这是真实的执行结果）。请基于这些真实结果继续下一步思考，**不要在代码执行前自行编造观察结果**。 + 3. 自验证： + - 关键事件（工具调用、检索结果、代码执行、准备最终回答）后，系统会进行显式自验证。 + - 如果自验证提示存在错误、证据不足、参数不完整或结果不可靠，必须优先修正、补充证据、重新调用工具，或清晰说明无法完成的部分。 + - 最终回答只有在自验证通过后才会展示给用户；如果系统返回 Verification feedback，请把它视为真实观察结果继续修正，不要忽略。 + 在思考结束后，当你认为可以回答用户问题，那么可以不生成代码，直接生成最终回答给到用户并停止循环。 生成最终回答时，你需要遵循以下规范： @@ -271,3 +276,13 @@ final_answer: 原始任务：{{task}} 请对迄今为止完成的工作进行清晰、简洁的总结。 + + +verification: + pre_messages: |- + 你是 ReAct 智能体的严格验证器。请仅根据任务、候选答案、工具输出和观察结果判断答案是否可靠，不要输出隐藏思维链。 + 你必须只输出 JSON。 + + post_messages: |- + 请验证候选答案是否覆盖用户意图、是否有观察结果支撑、是否处理了工具错误、引用是否可信、格式是否适合展示。 + 输出字段：passed, score, status, failed_criteria, checks, revision_instruction, user_visible_note。 diff --git a/backend/prompts/manager_system_prompt_template_en.yaml b/backend/prompts/manager_system_prompt_template_en.yaml index a830e007d..d44ed9a71 100644 --- a/backend/prompts/manager_system_prompt_template_en.yaml +++ b/backend/prompts/manager_system_prompt_template_en.yaml @@ -67,6 +67,11 @@ system_prompt: |- - Note that executed code is not visible to users. If users need to see the code, use 'code' for displaying code. - **IMPORTANT**: After code execution, the system will return content with "Observation:" marker (this is the real execution result). Please continue your next thinking based on these real results. **Do NOT fabricate observation results before code execution.** + 3. Self-verification: + - After critical events (tool calls, retrieval results, code execution, agent handoffs, and final-answer preparation), the system may run explicit verification. + - If verification reports errors, insufficient evidence, incomplete parameters, or unreliable results, you must repair the issue, gather more evidence, call tools again, or clearly state what cannot be completed. + - The final answer is shown to the user only after verification passes. If the system returns Verification feedback, treat it as a real observation and continue revising. + After thinking, when you believe you can answer the user's question, you can generate a final answer directly to the user without generating code and stop the loop. When generating the final answer, you need to follow these specifications: @@ -222,3 +227,13 @@ final_answer: Original task: {{task}} Please provide a clear and concise summary of the work completed so far. + + +verification: + pre_messages: |- + You are a strict verifier for a ReAct agent. Judge reliability only from the task, candidate answer, tool outputs, and observations. Do not output hidden chain-of-thought. + You must output JSON only. + + post_messages: |- + Verify whether the candidate answer covers the user's intent, is grounded in observations, handles tool errors, uses trustworthy citations, and is formatted for users. + Output fields: passed, score, status, failed_criteria, checks, revision_instruction, user_visible_note. diff --git a/backend/prompts/manager_system_prompt_template_zh.yaml b/backend/prompts/manager_system_prompt_template_zh.yaml index c58e6f2c9..a49ced82d 100644 --- a/backend/prompts/manager_system_prompt_template_zh.yaml +++ b/backend/prompts/manager_system_prompt_template_zh.yaml @@ -130,6 +130,11 @@ system_prompt: |- - 注意运行的代码不会被用户看到，所以如果用户需要看到代码，你需要使用'代码'表达展示代码。 - **重要**：代码执行后，系统会返回 "Observation:" 标记的内容（这是真实的执行结果）。请基于这些真实结果继续下一步思考，**不要在代码执行前自行编造观察结果**。 + 3. 自验证： + - 关键事件（工具调用、检索结果、代码执行、助手返回、准备最终回答）后，系统会进行显式自验证。 + - 如果自验证提示存在错误、证据不足、参数不完整或结果不可靠，必须优先修正、补充证据、重新调用工具，或清晰说明无法完成的部分。 + - 最终回答只有在自验证通过后才会展示给用户；如果系统返回 Verification feedback，请把它视为真实观察结果继续修正，不要忽略。 + 在思考结束后，当你认为可以回答用户问题，那么可以不生成代码，直接生成最终回答给到用户并停止循环。 生成最终回答时，你需要遵循以下规范： @@ -299,3 +304,13 @@ final_answer: 原始任务：{{task}} 请对迄今为止完成的工作进行清晰、简洁的总结。 + + +verification: + pre_messages: |- + 你是 ReAct 智能体的严格验证器。请仅根据任务、候选答案、工具输出和观察结果判断答案是否可靠，不要输出隐藏思维链。 + 你必须只输出 JSON。 + + post_messages: |- + 请验证候选答案是否覆盖用户意图、是否有观察结果支撑、是否处理了工具错误、引用是否可信、格式是否适合展示。 + 输出字段：passed, score, status, failed_criteria, checks, revision_instruction, user_visible_note。 diff --git a/backend/services/agent_service.py b/backend/services/agent_service.py index 4354a32d4..182000ec5 100644 --- a/backend/services/agent_service.py +++ b/backend/services/agent_service.py @@ -1142,6 +1142,7 @@ async def update_agent_info_impl(request: AgentInfoRequest, authorization: str = "prompt_template_name": prompt_template_name, "max_steps": request.max_steps, "provide_run_summary": request.provide_run_summary, + "verification_config": request.verification_config, "duty_prompt": request.duty_prompt, "constraint_prompt": request.constraint_prompt, "few_shots_prompt": request.few_shots_prompt, @@ -1667,6 +1668,7 @@ async def export_agent_by_agent_id( author=agent_info.get("author"), max_steps=agent_info["max_steps"], provide_run_summary=agent_info["provide_run_summary"], + verification_config=agent_info.get("verification_config"), duty_prompt=agent_info.get( "duty_prompt"), constraint_prompt=agent_info.get( @@ -1821,6 +1823,7 @@ async def import_agent_by_agent_id( "prompt_template_name": import_agent_info.prompt_template_name or SYSTEM_PROMPT_TEMPLATE_NAME, "max_steps": import_agent_info.max_steps, "provide_run_summary": import_agent_info.provide_run_summary, + "verification_config": getattr(import_agent_info, "verification_config", None), "duty_prompt": import_agent_info.duty_prompt, "constraint_prompt": import_agent_info.constraint_prompt, "few_shots_prompt": import_agent_info.few_shots_prompt, diff --git a/backend/utils/context_utils.py b/backend/utils/context_utils.py index 06e40f414..0c3af8915 100644 --- a/backend/utils/context_utils.py +++ b/backend/utils/context_utils.py @@ -611,6 +611,11 @@ def build_skeleton_execution_flow_component( lines.append(" - 注意运行的代码不会被用户看到，所以如果用户需要看到代码，你需要使用'代码'表达展示代码。") lines.append(" - **重要**：代码执行后，系统会返回 \"Observation:\" 标记的内容（这是真实的执行结果）。请基于这些真实结果继续下一步思考，**不要在代码执行前自行编造观察结果**。") lines.append("") + lines.append("3. 自验证：") + lines.append(" - 关键事件（工具调用、检索结果、代码执行、助手返回、准备最终回答）后，系统会进行显式自验证。") + lines.append(" - 如果自验证提示存在错误、证据不足、参数不完整或结果不可靠，必须优先修正、补充证据、重新调用工具，或清晰说明无法完成的部分。") + lines.append(" - 最终回答只有在自验证通过后才会展示给用户；如果系统返回 Verification feedback，请把它视为真实观察结果继续修正，不要忽略。") + lines.append("") lines.append("在思考结束后，当你认为可以回答用户问题，那么可以不生成代码，直接生成最终回答给到用户并停止循环。") lines.append("") lines.append("生成最终回答时，你需要遵循以下规范：") @@ -652,6 +657,11 @@ def build_skeleton_execution_flow_component( lines.append(" - Note that executed code is not visible to users. If users need to see the code, use 'code' for displaying code.") lines.append(" - **IMPORTANT**: After code execution, the system will return content with \"Observation:\" marker (this is the real execution result). Please continue your next thinking based on these real results. **Do NOT fabricate observation results before code execution.**") lines.append("") + lines.append("3. Self-verification:") + lines.append(" - After critical events (tool calls, retrieval results, code execution, agent handoffs, and final-answer preparation), the system may run explicit verification.") + lines.append(" - If verification reports errors, insufficient evidence, incomplete parameters, or unreliable results, you must repair the issue, gather more evidence, call tools again, or clearly state what cannot be completed.") + lines.append(" - The final answer is shown to the user only after verification passes. If the system returns Verification feedback, treat it as a real observation and continue revising.") + lines.append("") lines.append("After thinking, when you believe you can answer the user's question, you can generate a final answer directly to the user without generating code and stop the loop.") lines.append("") lines.append("When generating the final answer, you need to follow these specifications:") @@ -1325,4 +1335,4 @@ def build_app_context_string( Returns: Formatted app context string """ - return _format_app_context(app_name, app_description, user_id) \ No newline at end of file + return _format_app_context(app_name, app_description, user_id) diff --git a/doc/procedural-memory-verification.md b/doc/procedural-memory-verification.md index 69dbfa8fb..ea9f53290 100644 --- a/doc/procedural-memory-verification.md +++ b/doc/procedural-memory-verification.md @@ -81,10 +81,10 @@ The `_create_procedural_memory()` method exists in both `AsyncMemory` and `Memor **AsyncMemory signature:** ```python async def _create_procedural_memory( - self, - messages, - metadata=None, - llm=None, + self, + messages, + metadata=None, + llm=None, prompt=None ) ``` @@ -92,9 +92,9 @@ async def _create_procedural_memory( **Memory (sync) signature:** ```python def _create_procedural_memory( - self, - messages, - metadata=None, + self, + messages, + metadata=None, prompt=None ) ``` diff --git a/docker/init.sql b/docker/init.sql index 62b582f1f..046bdecf1 100644 --- a/docker/init.sql +++ b/docker/init.sql @@ -339,6 +339,7 @@ CREATE TABLE IF NOT EXISTS nexent.ag_tenant_agent_t ( is_new BOOLEAN DEFAULT FALSE, provide_run_summary BOOLEAN DEFAULT FALSE, enable_context_manager BOOLEAN DEFAULT FALSE, + verification_config JSONB, version_no INTEGER DEFAULT 0 NOT NULL, current_version_no INTEGER NULL, ingroup_permission VARCHAR(30), @@ -401,6 +402,7 @@ COMMENT ON COLUMN nexent.ag_tenant_agent_t.version_no IS 'Version number. 0 = dr COMMENT ON COLUMN nexent.ag_tenant_agent_t.current_version_no IS 'Current published version number. NULL means no version published yet'; COMMENT ON COLUMN nexent.ag_tenant_agent_t.ingroup_permission IS 'In-group permission: EDIT, READ_ONLY, PRIVATE'; COMMENT ON COLUMN nexent.ag_tenant_agent_t.enable_context_manager IS 'Whether to enable context management (compression) for this agent'; +COMMENT ON COLUMN nexent.ag_tenant_agent_t.verification_config IS 'Layered ReAct self-verification configuration'; COMMENT ON COLUMN nexent.ag_tenant_agent_t.greeting_message IS 'Agent greeting message displayed on chat initial screen'; COMMENT ON COLUMN nexent.ag_tenant_agent_t.example_questions IS 'List of example questions for starting a conversation with this agent'; diff --git a/docker/sql/v2.2.1_0601_add_agent_verification_config.sql b/docker/sql/v2.2.1_0601_add_agent_verification_config.sql new file mode 100644 index 000000000..d3882e1e2 --- /dev/null +++ b/docker/sql/v2.2.1_0601_add_agent_verification_config.sql @@ -0,0 +1,7 @@ +-- Migration: Add layered ReAct self-verification config to agents +-- Description: Stores per-agent verification controls for step-level and final-answer validation. + +ALTER TABLE nexent.ag_tenant_agent_t +ADD COLUMN IF NOT EXISTS verification_config JSONB; + +COMMENT ON COLUMN nexent.ag_tenant_agent_t.verification_config IS 'Layered ReAct self-verification configuration'; diff --git a/frontend/app/[locale]/agents/components/agentInfo/AgentGenerateDetail.tsx b/frontend/app/[locale]/agents/components/agentInfo/AgentGenerateDetail.tsx index 58ed2e0c6..24ec60616 100644 --- a/frontend/app/[locale]/agents/components/agentInfo/AgentGenerateDetail.tsx +++ b/frontend/app/[locale]/agents/components/agentInfo/AgentGenerateDetail.tsx @@ -21,6 +21,7 @@ import { Textarea } from "@/components/ui/textarea"; import { AgentConfigUpdate, + DEFAULT_AGENT_VERIFICATION_CONFIG, PromptTemplate, } from "@/types/agentConfig"; import { @@ -170,6 +171,7 @@ export default function AgentGenerateDetail({}) { constraintPrompt: editedAgent.constraint_prompt || "", fewShotsPrompt: editedAgent.few_shots_prompt || "", provideRunSummary: editedAgent.provide_run_summary || false, + verificationEnabled: editedAgent.verification_config?.enabled ?? true, businessDescription: editedAgent.business_description || "", businessLogicModelName:editedAgent.business_logic_model_name, businessLogicModelId: editedAgent.business_logic_model_id, @@ -807,7 +809,7 @@ export default function AgentGenerateDetail({}) { - + - + + + +