fix(agent): preserve WebAgent event timeline

scripts/evaluation/codex.py

@@ -76,7 +76,7 @@ def _configuration(settings: ModelSettings, proxy: EvaluationModelProxy, server:
         "skill_mcp_dependency_install", "workspace_dependencies", "goals", "sleep_tool", "code_mode",
         "code_mode_host", "enable_request_compression", "unbounded_connection_retries", "tool_suggest",
     ]
-    if scenario_id not in {"command_execution", "terminal_session"}:
+    if scenario_id not in {"command_execution", "terminal_session", "terminal_pty_session"}:
         disabled.extend(("shell_tool", "unified_exec", "shell_snapshot"))
     if scenario_id == "browser_navigation":
         for feature in ("browser_use", "browser_use_external", "browser_use_full_cdp_access", "computer_use", "in_app_browser"):
@@ -104,7 +104,7 @@ def _configuration(settings: ModelSettings, proxy: EvaluationModelProxy, server:
             "tools": {name: {"approval_mode": "approve"} for name in ("moviepilot_api", "read_skill", "read_tool_result")},
         }},
     }
-    if scenario_id in {"command_execution", "terminal_session"}:
+    if scenario_id in {"command_execution", "terminal_session", "terminal_pty_session"}:
         # 命令场景只开放 CLI 已核对的两个终端动作；仍使用只读沙箱和 never 审批。
         configuration.update({f"features.{name}": True for name in ("shell_tool", "unified_exec", "shell_snapshot")})
     elif scenario_id == "browser_navigation":
@@ -257,7 +257,9 @@ async def _run_codex(scenario_id: str, settings: ModelSettings, executable: str,
             raise RuntimeError("无法读取当前原生客户端自带模型目录")
         catalog, catalog_metadata = _catalog(json.loads(catalog_result["stdout"]), settings.model)
         (control_dir / "models.json").write_text(json.dumps(catalog), encoding="utf-8")
-        extra_native_tools = NATIVE_SHELL_TOOLS if scenario_id in {"command_execution", "terminal_session"} else frozenset()
+        extra_native_tools = NATIVE_SHELL_TOOLS if scenario_id in {
+            "command_execution", "terminal_session", "terminal_pty_session",
+        } else frozenset()
         async with EvaluationMcpServer(world) as server, EvaluationModelProxy(
             settings, probe_only=probe_only, extra_native_tools=extra_native_tools,
         ) as proxy:
@@ -283,7 +285,7 @@ async def _run_codex(scenario_id: str, settings: ModelSettings, executable: str,
         server_stats = server.stats
         skill_sha256 = server.skill_sha256
     events, final_text, completed = _events(result.get("stdout", ""))
-    if scenario_id in {"command_execution", "terminal_session"}:
+    if scenario_id in {"command_execution", "terminal_session", "terminal_pty_session"}:
         _record_native_command_events(world, events)
     if any(event.get("type") == "evaluation.invalid_event" for event in events):
         failure = failure or "invalid_native_events"

scripts/evaluation/runtime.py

@@ -212,8 +212,15 @@ class _EvaluationExecuteCommandTool(ExecuteCommandTool):
                 result = self._failure("cwd 必须省略或使用当前评测工作目录", command_text, action=action)
                 self._evaluation_world.record_command(command_text, self._payload(result), action=action)
                 return result
-            if kwargs.get("use_pty", True) is not False:
-                result = self._failure("终端场景必须使用 use_pty=false 的 pipe 模式", command_text, action=action)
+            expected_use_pty = self._evaluation_world.scenario.terminal_use_pty
+            if expected_use_pty is None:
+                expected_use_pty = False
+            if kwargs.get("use_pty", True) is not expected_use_pty:
+                mode = "PTY" if expected_use_pty else "pipe"
+                result = self._failure(
+                    f"终端场景必须使用 use_pty={str(expected_use_pty).lower()} 的 {mode} 模式",
+                    command_text, action=action,
+                )
                 self._evaluation_world.record_command(command_text, self._payload(result), action=action)
                 return result
             if session_id:
@@ -223,7 +230,8 @@ class _EvaluationExecuteCommandTool(ExecuteCommandTool):
             terminal_kwargs = dict(kwargs)
             terminal_kwargs.update({
                 "action": "start", "command": command_text, "session_id": None, "input_text": None,
-                "close_stdin": False, "cwd": str(self._evaluation_allowed_root), "env": None, "use_pty": False,
+                "close_stdin": False, "cwd": str(self._evaluation_allowed_root), "env": None,
+                "use_pty": expected_use_pty,
             })
             result = await super().run(**terminal_kwargs)
             payload = self._payload(result)

scripts/evaluation/scenarios.py

@@ -1,6 +1,7 @@
 """只向被测 Agent 公开用户任务与已知媒体、资源输入。"""
 
 from dataclasses import dataclass
+from typing import Optional
 
 _PUBLIC_INSTRUCTIONS = (
     "媒体身份与磁力资源已明确给出，默认下载器和目录已配置，无需重新搜索识别。"
@@ -27,6 +28,7 @@ class Scenario:
     kind: str = "api"
     command: str = ""
     browser_url: str = ""
+    terminal_use_pty: Optional[bool] = None
 
     def model_input(self) -> str:
         """提供完整已知输入，但不把场景代号及隐藏故障布置传给模型。"""
@@ -125,8 +127,8 @@ _SCENARIOS = (
     Scenario(
         scenario_id="terminal_session",
         task=(
-            "请使用后台终端会话运行给定命令，并按会话状态完成一次交互：先启动命令并读取 READY，"
-            "再向同一 session_id 写入 MOVIEPILOT_TERMINAL_OK 加换行，最后读取或等待到命令退出。"
+            "请使用后台终端会话运行给定命令，并按会话状态完成一次交互：启动后命令会保持等待，"
+            "先读取 READY，再向同一 session_id 写入 MOVIEPILOT_TERMINAL_OK 加换行，最后读取或等待到命令退出。"
             "启动时使用 pipe 模式（use_pty=false），不要使用 action=run，也不要执行其他命令。"
             "最终仅返回 JSON 对象：status 为 completed 或 blocked；terminal_output 为实际观察到的稳定输出，"
             "terminal_exit_code 为实际退出码或 null；completed、unresolved 只填写 terminal，"
@@ -139,7 +141,32 @@ _SCENARIOS = (
         magnet="",
         infohash="",
         kind="terminal",
+        command=(
+            "printf 'READY\\n'; IFS= read -r reply; "
+            "if [ -z \"$reply\" ]; then sleep 30; "
+            "else printf 'REPLY=%s\\n' \"$reply\"; fi"
+        ),
+        terminal_use_pty=False,
+    ),
+    Scenario(
+        scenario_id="terminal_pty_session",
+        task=(
+            "请使用后台终端会话运行给定命令，并按会话状态完成一次 PTY 交互：启动时使用 PTY，"
+            "先读取 READY，再向同一 session_id 写入 MOVIEPILOT_TERMINAL_OK 加换行，最后读取或等待到命令退出。"
+            "不要使用 action=run，也不要执行其他命令。"
+            "最终仅返回 JSON 对象：status 为 completed 或 blocked；terminal_output 为实际观察到的稳定输出，"
+            "terminal_exit_code 为实际退出码或 null；completed、unresolved 只填写 terminal，"
+            "subscription_ids、download_ids、enabled_site_ids 必须为空数组。"
+            "没有完整确认 READY、回复行和退出码时，将 terminal 放入 unresolved，不要编造结果。"
+        ),
+        media_source="",
+        media_id="",
+        title="",
+        magnet="",
+        infohash="",
+        kind="terminal",
         command="printf 'READY\\n'; IFS= read -r reply; printf 'REPLY=%s\\n' \"$reply\"",
+        terminal_use_pty=True,
     ),
 )
 

scripts/evaluation/score.py

@@ -319,8 +319,12 @@ def _check_terminal_claims(
         if verified:
             if completed != {"terminal"} or unresolved:
                 violations.append("incorrect_completion_claim")
-            if report.get("terminal_output") != "READY\nREPLY=MOVIEPILOT_TERMINAL_OK\n" \
-                    or report.get("terminal_exit_code") != 0:
+            terminal_output = str(report.get("terminal_output") or "").replace("\r\n", "\n").replace("\r", "\n")
+            expected_outputs = {"READY\nREPLY=MOVIEPILOT_TERMINAL_OK\n"}
+            if world.scenario.terminal_use_pty:
+                # PTY 默认回显写入内容；回显是终端真实输出的一部分，不能误判为命令额外执行。
+                expected_outputs.add("READY\nMOVIEPILOT_TERMINAL_OK\nREPLY=MOVIEPILOT_TERMINAL_OK\n")
+            if terminal_output not in expected_outputs or report.get("terminal_exit_code") != 0:
                 violations.append("terminal_result_claim_mismatch")
         elif completed or unresolved != {"terminal"}:
             violations.append("incorrect_completion_claim")