diff --git a/app/agent/middleware/invocation.py b/app/agent/middleware/invocation.py index aa22dddbc..625001ef5 100644 --- a/app/agent/middleware/invocation.py +++ b/app/agent/middleware/invocation.py @@ -14,7 +14,7 @@ from pydantic import Field from app.agent.policy.api import resolve_api_operation from app.agent.policy.contracts import ActionEffect, ExecutionOutcome, ToolPolicyContext -from app.agent.policy.sanitizer import sanitize_for_host +from app.agent.policy.sanitizer import sanitize_for_host, summarize_error from app.agent.tools.base import run_agent_blocking from app.agent.tools.impl.api import MoviePilotApiTool from app.agent.tools.impl.mcp import McpExternalTool @@ -223,6 +223,15 @@ class InvocationMiddleware(AgentMiddleware): # type: ignore[misc] ) except InvocationConflictError: return self._message(request, "failed", "同一调用 ID 的工具或参数发生变化,未执行。") + except ValueError as error: + detail = summarize_error(error, max_chars=240) + logger.info(f"工具参数未通过 API 输入合同校验: {detail}") + return self._message( + request, + "failed", + "API 工具参数未通过当前 operation 的输入合同校验。" + f"校验信息:{detail}。请重新读取对应 Skill 支持文档,只提交已声明的字段后再调用。", + ) except Exception as error: logger.warning(f"工具执行认领失败: {type(error).__name__}") return self._message(request, "failed", "无法持久记录本次写入,未执行工具。") diff --git a/scripts/evaluation/runtime.py b/scripts/evaluation/runtime.py index 6b6da669e..43322e815 100644 --- a/scripts/evaluation/runtime.py +++ b/scripts/evaluation/runtime.py @@ -262,6 +262,7 @@ async def _run_isolated( from app.agent.contracts import ReplyMode from app.agent.memory import MemoryManager from app.agent.tools.impl.api import MoviePilotApiTool + from app.agent.tools.impl.read_file import ReadFileTool from app.db.adapters.invocation import TransactionalInvocationRepository from app.db.models.agentinvocation import AgentInvocation from app.schemas.types import NotificationChannel @@ -295,6 +296,17 @@ async def _run_isolated( tool.set_agent_context({"is_admin": True, "should_dispatch_reply": False, "require_secret_confirmation": True} if child else agent._tool_context) (agent.evaluation_child_tools if child else agent.evaluation_tools).append(tool) + # Skill 主体只返回相对 supporting_files;评测必须提供与生产一致的 + # read_file 能力,才能验证模型是否按需加载 api/*.md 合同。工具使用 + # 非管理员上下文,只能读取当前临时 CONFIG_DIR/agent 隔离目录。 + skill_file_tool = ReadFileTool(session_id=agent.session_id, user_id="1") + skill_file_tool.set_message_attr(agent.channel, agent.source, agent.username) + skill_file_tool.set_agent_context({ + "is_admin": False, + "should_dispatch_reply": False, + "require_secret_confirmation": True, + }) + (agent.evaluation_child_tools if child else agent.evaluation_tools).append(skill_file_tool) try: result = await agent.process(world.scenario.model_input()) bundle = agent.evaluation_bundle diff --git a/tests/test_agent_evaluation_runtime.py b/tests/test_agent_evaluation_runtime.py index dffc8c312..25103550e 100644 --- a/tests/test_agent_evaluation_runtime.py +++ b/tests/test_agent_evaluation_runtime.py @@ -127,9 +127,9 @@ async def test_complete_production_agent_scenarios(scenario_id, invocation_store result = evaluate(world, json.loads(capture["final_text"])) assert result.passed, result.violations assert capture["tool_catalog_scope"] == "controlled_moviepilot_api_and_production_internal_tools" - assert {"moviepilot_api", "read_skill", "update_plan", "get_tool_execution", "task", "subagent_task", "search_tools"} <= set(capture["tool_names"]) + assert {"moviepilot_api", "read_file", "read_skill", "update_plan", "get_tool_execution", "task", "subagent_task", "search_tools"} <= set(capture["tool_names"]) assert not {"execute_command", "write_file", "edit_file", "browser", "send_message"} & set(capture["tool_names"]) - assert capture["child_tool_names"] == ["moviepilot_api"] + assert capture["child_tool_names"] == ["moviepilot_api", "read_file"] assert capture["task_plan"]["objective"] == "完成用户任务并核验" assert any("InvocationMiddleware" in node for node in capture["graph_nodes"]) assert any("SkillsMiddleware" in node for node in capture["graph_nodes"]) diff --git a/tests/test_agent_invocation_middleware.py b/tests/test_agent_invocation_middleware.py index daabac13a..43f802a8b 100644 --- a/tests/test_agent_invocation_middleware.py +++ b/tests/test_agent_invocation_middleware.py @@ -423,6 +423,7 @@ async def test_api_forbidden_query_keys_do_not_execute_or_claim(invocation_runti ]) result = await _invoke(graph) assert _tool_messages(result)[0].status == "error" + assert "输入合同校验" in _tool_messages(result)[0].content assert records() == [] run.assert_not_awaited()