From ee52ca0705083a5d9a727b2432e5005b837402be Mon Sep 17 00:00:00 2001 From: jxxghp Date: Sat, 12 Sep 2026 00:50:00 +0800 Subject: [PATCH] test(agent): wire native browser harness --- docs/refactor/agent-codex-parity-plan.md | 7 +- scripts/evaluation/codex.py | 282 ++++++++++++++++++----- scripts/evaluation/proxy.py | 25 +- tests/test_agent_evaluation_codex.py | 41 ++++ 4 files changed, 287 insertions(+), 68 deletions(-) diff --git a/docs/refactor/agent-codex-parity-plan.md b/docs/refactor/agent-codex-parity-plan.md index ef35c59f8..bd5f82ab6 100644 --- a/docs/refactor/agent-codex-parity-plan.md +++ b/docs/refactor/agent-codex-parity-plan.md @@ -2,7 +2,7 @@ > 归档位置：`docs/refactor/`。 -更新时间：2026-09-11 +更新时间：2026-09-12 这份文档是 Agent 能力对齐的交付路线和每轮验收合同。它记录当前证据与未完成目标，不把一次成功的模型调用当成“已经和 Codex 一样聪明”。最终判断必须同时看模型行为、工具执行、任务生命周期、业务终态和失败后的收敛结果。 @@ -22,7 +22,7 @@ Google 的 OpenAI-compatible 层会丢失 Gemini 3 工具回复中的 `thought_s | C1.3 终端任务作用域 | 已完成 | 终端归属由宿主对象身份决定；定时运行、会话、子任务和内部工具管理器隔离；封口先于清理，排队或运行中的命令都不会在任务结束后迟到启动 | | S2.3 运行中消息排队与 WebAgent 输入 | 已完成本轮实现并修复丢消息、时序错位和 continuation 状态不刷新的遗漏 | 运行中仍可提交新消息；消息按会话原子入队，在下一次模型调用边界注入真实 `HumanMessage`；SSE 报告 queued/applied，稳定 `steering_message_id` 贯穿展示快照、前端本地状态和恢复合并；应用点收口当前助手并创建 continuation 助手，用户气泡落在真实事件边界；工具按 `tool_id` 报告 running/done/error；停止后不再派发后续工具 | | C1.5 长上下文压缩与任务约束保留 | 同条件配对已通过 | 真实压缩前后预算约 69.5K → 26.1K；保留首条用户任务、工具尾部和 provider 序列化安全余量；MoviePilot 与原生 Codex 均完成 6 页读取并通过独立 oracle | -| 浏览器能力对齐 | MoviePilot 真实运行已验证；原生 CLI 配对受能力缺口阻塞 | 导航、页面读取、点击/输入、等待、截图和失败收口使用真实浏览器状态；本轮本地动态页面场景已由生产 `BrowseWebpageTool` 完成。`codex features list` 虽显示 `browser_use` 为 stable，但真实受控 `codex exec 0.153.4` 请求仍只保留计划、工具搜索和评测 MCP；浏览器场景 12 次模型调用后没有产生浏览器账本事件并耗尽预算，feature flag 不能替代原生 browser plugin/host，保留为 Harness 能力缺口，不能伪造配对 | +| 浏览器能力对齐 | MoviePilot 真实运行已验证；原生 CLI 已接入真实 Browser plugin，但回环导航被浏览器保存的用户权限阻断 | 导航、页面读取、点击/输入、等待、截图和失败收口使用真实浏览器状态；本轮本地动态页面场景已由生产 `BrowseWebpageTool` 完成。原生 `codex exec 0.153.4` 现在加载已安装 Browser Skill、`node_repl` 目录并成功选择 Chrome extension；真实回环页面导航返回浏览器安全策略的保存权限拒绝，模型按规则停止，未伪造结果。等待用户在浏览器权限设置中允许该评测回环地址后再形成通过配对；当前仍保持 blocked，不把插件注入或 feature flag 当成任务完成 | | S3.1 通用子代理 | 三轮 held-out 配对通过，仍需终端分享/取消边界收益数据 | 主 Agent 只暴露并派发 `general-purpose`；旧的专用画像已删除，不保留兼容入口。held-out 场景已在三轮独立运行中验证两个并行只读子任务、授权、工具角色和零副作用；仍需终端分享/取消边界的收益数据 | ## 每轮硬门禁 @@ -51,7 +51,7 @@ uv run --locked --no-sync python -m scripts.evaluation \ ## 工具与 Harness 对齐清单 - **命令行**：普通一次性命令与后台终端分别支持 pipe/PTY、共享 cwd/shell/login/环境和 UTF-8；输入写入、空写入、EOF、分页、短写、interrupt、kill、超时、取消和进程组收尾都有结构化终态。`command_execution` 与终端场景只在原生 CLI 真实广告 `functions.exec_command`/`functions.write_stdin` 时按场景开启，并把 MoviePilot 生产 `ExecuteCommandTool` 绑定到临时目录；终端场景的原生评测使用 app-server，记录命令输出增量和终端输入事件。生产与原生 PTY 已通过同条件真实配对；原生 pipe 仍因 CLI 在一次性命令后关闭 stdin 而没有输入事件，保持 blocked，不把适配器能力误算成原生 Harness 对齐。适配器将单请求、代理和 app-server 流式空闲时间统一跟随评测墙钟预算，避免较长推理被 Harness 的硬编码超时误判。 -- **浏览器**：工具目录必须明确导航、读取、交互、等待和截图的动作与权限；浏览器会话、页面状态和失败重试由宿主持有，不能由模型字符串冒领。新增 `browser_navigation` 场景使用回环动态页面验证生产工具；原生 CLI 探针没有广告浏览器工具，原生配对保持 blocked，等待可用的 Codex browser harness。 +- **浏览器**：工具目录必须明确导航、读取、交互、等待和截图的动作与权限；浏览器会话、页面状态和失败重试由宿主持有，不能由模型字符串冒领。`browser_navigation` 场景使用回环动态页面验证生产工具；原生 CLI 适配器只在本机 Browser plugin、`node_repl` 和 Skill 文件均存在时开启对应目录，并把可移植的 `` 展开为受信绝对路径。真实原生运行已选择 Chrome extension，但回环导航被保存的浏览器权限拒绝，模型按安全规则停止，保持 blocked；权限放开后才能继续形成同场景通过配对。 - **子代理**：主 Agent 自动选择通用子代理；专用画像只有在 held-out 任务上证明提高成功率、减少调用或降低副作用风险时才保留。子代理不能自行发送消息、执行高影响写操作或继承兄弟任务句柄。 - **API Skill 与合同读取**：`skills/moviepilot-api/SKILL.md` 只负责路由索引和工作流；`api/*.md` 各自包含完整操作合同及该类别需要的 Body Models，不再依赖单独的 `api/models.md`。真实 Agent 先用 `read_skill` 得到 supporting-file 清单，再用同一个工具的 `file=api/.md` 参数按需读取类别文档；评测目录必须提供同一条链路，不能只返回文件名而不给模型读取能力。operation 输入错误还要返回该 operation 的允许字段、必填字段和类型约束，帮助模型纠正后重试。 - **长任务与新消息**：入站消息在任务运行时仍可接受并进入有界队列；应用到下一模型边界时必须保留 tool-call/tool-result 配对和取消语义，不能只把文本拼到系统提示词。WebAgent 将工具开始/完成/失败作为带稳定 `tool_id` 的结构化事件；前端按事件序列维护每个工具状态，queued ACK 只建立排队气泡，后续事件继续归属于当前助手段，直到 applied 事件报告真实模型边界后再创建 continuation，避免“最新工具覆盖全部状态”或把用户消息挪到顶部。 @@ -75,6 +75,7 @@ uv run --locked --no-sync python -m scripts.evaluation \ - 本轮第二阶段：用同一 `gpt-5.6-luna + max`、同一场景和预算显式启用 Codex OAuth，首次生成有效成对摘要 `/tmp/moviepilot-agent-round-luna-oauth-dedup-pair.json`。MoviePilot live `/tmp/moviepilot-agent-round-luna-oauth-live-dedup-5.json` 因最终 infohash 少两位而未通过；原生 Codex `/tmp/moviepilot-agent-round-luna-oauth-native-dedup-8.json` 通过。摘要 `pair_valid=true`、`both_passed=false`，证明真实生产图与 Codex 仍有可观测的终态可靠性差距；下一轮先针对精确最终报告和未知结果收口做多次重复，再扩展浏览器、终端和中途消息场景。 - 本轮第三阶段：修正原生动态工具搜索对 `multi_agent_v1` 协作命名空间的投影，正向保留 `wait_agent`、`resume_agent` 等原生控制动作，负向继续拒绝 `read_file`。在同一 Harness（`71be4c436acc5dc2d5a0cd6996c3ca10688a9c4ba98fd11594e536f7aa440c22`）下，`dedup_existing` 配对 `/tmp/moviepilot-agent-round-luna-oauth-dedup-pair-13.json` 与 `unknown_download` 配对 `/tmp/moviepilot-agent-round-luna-oauth-unknown-pair-14.json` 均 `pair_valid=true`、`both_passed=true`；MoviePilot 分别使用 6/6 次模型调用，原生 Codex 使用 10/14 次。`honest_unknown` 配对 `/tmp/moviepilot-agent-round-luna-oauth-honest-pair-16.json` 仍为 `both_passed=false`：MoviePilot 在 9 次调用后如实保留下载未核验，原生 Codex 达到 16 次上限后没有最终 JSON，产生 `invalid_final_report`。这保留了真实失败边界，不能等同整体 Codex 智能已完成。 - 命令行与浏览器能力阶段：`command_execution` 在同一 `gpt-5.6-luna + max` 和独立 Codex OAuth 下已形成真实配对 `/tmp/moviepilot-agent-round-luna-oauth-command-pair-final.json`，`pair_valid=true`、`both_passed=true`，harness 为 `c409ee1ca7899ccfcefd38832d6a343f024f8fc67fb65f14c3eec08552203257`。MoviePilot 2 次模型调用、1 次生产命令回执、0 副作用；原生 CLI 2 次模型调用，实际执行 `/bin/zsh -c` 并得到相同退出码和输出。生产 stdout 的 `[标准输出]` 展示包装已在独立判定器中与命令内容分离。 +- 原生浏览器 Harness 补充验证：适配器接入本机已安装的 Browser plugin、`node_repl` 和完整 Browser Skill，并把 `` 展开到临时受信文件；动态回环 fixture 会通过 `/clicked` 回调记录真实按钮点击。`/tmp/moviepilot-agent-round-luna-oauth-native-browser-expanded-20260912.json` 中模型 6 次调用后成功选择 Chrome extension，但 `goto` 和带 `allow_private_network=true` 的重试均被浏览器保存权限拒绝，模型正确返回 `blocked`，没有产生伪造账本事件。原生浏览器能力已经可被 Harness 发现，任务配对仍因外部保存权限保持 blocked。 - 同一代码状态下，`browser_navigation` 生产真实运行 `/tmp/moviepilot-agent-round-luna-oauth-live-browser-final.json` 通过：6 次模型调用、4 次浏览器回执，真实页面点击后观察到 `BROWSER_OK`。`codex exec 0.153.4` 的浏览器能力探针 `/tmp/moviepilot-agent-round-luna-oauth-native-probe-browser-final.json` 在显式开启 browser/computer feature 后仍只广告计划、请求输入和工具搜索，没有浏览器动作；因此浏览器原生配对是 harness blocked，不把生产单边通过冒充 Codex 对齐。 - `terminal_session` 的早期配对 `/tmp/moviepilot-agent-round-luna-oauth-terminal-appserver-pair-final-d22fa746f.json` 保留了原生 pipe 在 READY 前关闭 stdin 的失败证据。随后评测适配器改用与整轮预算一致的流式超时，仍不掩盖该一次性 pipe 交互边界；生产路径的持续会话测试保持通过。 - 同条件的 `terminal_pty_session` 配对 `/tmp/moviepilot-agent-round-luna-oauth-terminal-pty-appserver-pair-timeout180-sleep1-ready1-20260911.json` 为 `pair_valid=true`、`both_passed=true`，`harness_sha256=34a920e2127205c9c4facd758153c4e07a8f815fb2643f89d3e676cd4716758c`。MoviePilot `/tmp/moviepilot-agent-round-luna-oauth-live-terminal-pty-timeout180-sleep1-ready1-20260911.json` 4 次模型调用、3 次真实终端操作；原生 app-server `/tmp/moviepilot-agent-round-luna-oauth-native-terminal-pty-appserver-timeout180-sleep1-ready1-20260911.json` 3 次模型调用，均由独立账本确认 PTY 输入、`READY`/回复输出和退出码 0。PTY 探针命令在输出 READY 前保留 1 秒启动窗口、回复后保留 1 秒收尾窗口，模拟真实服务启动和终端交接，避免丢失进程早期输出。 diff --git a/scripts/evaluation/codex.py b/scripts/evaluation/codex.py index c5df92063..98d7c05a2 100644 --- a/scripts/evaluation/codex.py +++ b/scripts/evaluation/codex.py @@ -10,12 +10,15 @@ import signal import subprocess import tempfile import time +from contextlib import contextmanager, nullcontext +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path +from threading import Thread from typing import Any from scripts.evaluation.live import _parse_final, _validate_report, _worker_environment from scripts.evaluation.models import ModelSettings -from scripts.evaluation.proxy import NATIVE_SHELL_TOOLS, EvaluationModelProxy +from scripts.evaluation.proxy import NATIVE_BROWSER_TOOLS, NATIVE_SHELL_TOOLS, EvaluationModelProxy from scripts.evaluation.score import evaluate from scripts.evaluation.server import EvaluationMcpServer from scripts.evaluation.world import EvaluationWorld @@ -23,6 +26,129 @@ from scripts.evaluation.world import EvaluationWorld SUPPORTED_CLI_VERSION = "codex-cli 0.153.4" MAX_PROCESS_OUTPUT_BYTES = 8 * 1024 * 1024 NATIVE_TERMINAL_SCENARIOS = frozenset({"terminal_session", "terminal_pty_session"}) +NATIVE_BROWSER_SCENARIOS = frozenset({"browser_navigation"}) + + +def _browser_runtime_configuration(executable: str, instruction_dir: Path) -> dict[str, Any] | None: + """发现本机已安装的浏览器插件运行时，缺少任一受信文件就保持原生边界关闭。""" + codex_home = Path(os.environ.get("CODEX_HOME") or Path.home() / ".codex") + plugin_root = Path(os.environ.get( + "MOVIEPILOT_EVAL_BROWSER_PLUGIN_ROOT", + str(codex_home / ".tmp" / "bundled-marketplaces" / "openai-bundled" / "plugins" / "browser"), + )) + node_repl = Path(os.environ.get( + "MOVIEPILOT_EVAL_NODE_REPL", + "/Applications/ChatGPT.app/Contents/Resources/cua_node/bin/node_repl", + )) + node_path = Path(os.environ.get("MOVIEPILOT_EVAL_NODE_PATH", str(node_repl.parent / "node"))) + node_modules = Path(os.environ.get( + "MOVIEPILOT_EVAL_NODE_MODULES", + str(node_repl.parent.parent / "lib" / "node_modules"), + )) + browser_service = os.environ.get("MOVIEPILOT_EVAL_BROWSER_SERVICE") + service_candidates = ([Path(browser_service)] if browser_service else []) + sorted( + (codex_home / "plugins" / "cache" / "openai-bundled" / "browser").glob( + "*/scripts/browser-service.mjs" + ), + reverse=True, + ) + browser_service_path = next((path for path in service_candidates if path.is_file()), None) + browser_client = plugin_root / "scripts" / "browser-client.mjs" + if (not node_repl.is_file() or not node_path.is_file() or not node_modules.is_dir() + or not browser_client.is_file() or browser_service_path is None): + return None + marketplace = plugin_root.parent.parent + client_hashes = { + hashlib.sha256(browser_client.read_bytes()).hexdigest(), + } + chrome_client = codex_home / "plugins" / "cache" / "openai-bundled" / "chrome" / "latest" / "scripts" / "browser-client.mjs" + if chrome_client.is_file(): + client_hashes.add(hashlib.sha256(chrome_client.read_bytes()).hexdigest()) + trusted_paths = ":".join(str(path) for path in (codex_home, node_modules, plugin_root)) + node_environment = { + "NODE_REPL_NATIVE_PIPE_CONNECT_TIMEOUT_MS": "1000", + "NODE_REPL_NODE_MODULE_DIRS": str(node_modules), + "NODE_REPL_NODE_PATH": str(node_path), + "NODE_REPL_TRUSTED_CODE_PATHS": trusted_paths, + "NODE_REPL_TRUSTED_BROWSER_CLIENT_SHA256S": ",".join(sorted(client_hashes)), + "NODE_REPL_TRUSTED_SERVICES": json.dumps( + {"browser": str(browser_service_path), "sky": "@oai/sky/service"}, ensure_ascii=False + ), + "CODEX_HOME": str(codex_home), + "BROWSER_USE_AVAILABLE_BACKENDS": "chrome,iab", + "BROWSER_USE_TINYSKY_ENABLED": "1", + "BROWSER_USE_CODEX_APP_BUILD_FLAVOR": "prod", + "BROWSER_USE_CODEX_APP_VERSION": browser_service_path.parent.parent.name, + "BROWSER_USE_BROWSER_CLIENT_BUILD": browser_service_path.parent.parent.name, + "CODEX_CLI_PATH": executable, + } + # The installed skill intentionally uses the portable ```` token. + # A standalone CLI run has no desktop plugin context to expand it, so create + # a private copy with the same instructions and the verified absolute root. + skill = plugin_root / "skills" / "control-in-app-browser" / "SKILL.md" + expanded_skill = instruction_dir / "browser-control-skill.md" + expanded_skill.write_text(skill.read_text(encoding="utf-8").replace("", str(plugin_root)), encoding="utf-8") + return { + "node_repl": { + "command": str(node_repl), "args": [], "enabled": True, "required": False, + "startup_timeout_sec": 120, "tool_timeout_sec": 120, + "default_tools_approval_mode": "approve", "env": node_environment, + }, + "marketplace": {"source_type": "local", "source": str(marketplace)}, + "plugins": {"browser@openai-bundled": {"enabled": True}, "chrome@openai-bundled": {"enabled": True}}, + "skill": str(expanded_skill), + } + + +@contextmanager +def _native_browser_fixture() -> Any: + """启动只绑定回环地址的动态测试页，并记录真实浏览器的点击回调。""" + state = {"clicked": False} + + class _Handler(BaseHTTPRequestHandler): + """提供最小页面和点击回调，不记录请求正文或客户端环境。""" + + def do_GET(self) -> None: # noqa: N802 - 标准库处理器方法名 + """返回测试页面或确认按钮回调，其他路径返回 404。""" + if self.path == "/fixture": + body = ( + "MoviePilot Browser Fixture" + "

MoviePilot Browser Fixture

" + "

PENDING

" + "" + "
" + ).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + return + if self.path == "/clicked": + state["clicked"] = True + body = b"ok" + self.send_response(204) + self.send_header("Cache-Control", "no-store") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + return + self.send_error(404) + + def log_message(self, _format: str, *_args: Any) -> None: + """禁止标准库把本地测试请求写入 stderr。""" + + server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler) + thread = Thread(target=server.serve_forever, name="moviepilot-native-browser", daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}/fixture", state + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) def _catalog(payload: Any, model: str) -> tuple[dict[str, Any], dict[str, Any]]: @@ -68,7 +194,8 @@ def _redact(value: Any, tokens: tuple[str, ...]) -> Any: def _configuration(settings: ModelSettings, proxy: EvaluationModelProxy, server: EvaluationMcpServer, - control_dir: Path, *, scenario_id: str = "") -> dict[str, Any]: + control_dir: Path, *, scenario_id: str = "", + browser_runtime: dict[str, Any] | None = None) -> dict[str, Any]: """显式隔离原生客户端的外部能力，仍保留其默认提示词、计划及子代理循环。""" disabled = [ "apps", "browser_use", "browser_use_external", "browser_use_full_cdp_access", "computer_use", @@ -109,10 +236,23 @@ def _configuration(settings: ModelSettings, proxy: EvaluationModelProxy, server: # 命令场景只开放 CLI 已核对的两个终端动作；仍使用只读沙箱和 never 审批。 configuration.update({f"features.{name}": True for name in ("shell_tool", "unified_exec", "shell_snapshot")}) elif scenario_id == "browser_navigation": - # 让探针核对 CLI 自身是否广告浏览器能力；当前版本未广告时必须留在差异证据中。 + # 浏览器插件只能在本机受信运行时完整存在时开启，缺失时保留 blocked 证据。 configuration.update({f"features.{name}": True for name in ( "browser_use", "browser_use_external", "browser_use_full_cdp_access", "computer_use", "in_app_browser", )}) + if browser_runtime is not None: + configuration.update({ + "features.plugins": True, + "skills.bundled.enabled": True, + "skills.include_instructions": False, + "mcp_servers": {**configuration["mcp_servers"], "node_repl": browser_runtime["node_repl"]}, + "marketplaces.openai-bundled": browser_runtime["marketplace"], + "plugins.browser@openai-bundled": browser_runtime["plugins"]["browser@openai-bundled"], + "plugins.chrome@openai-bundled": browser_runtime["plugins"]["chrome@openai-bundled"], + # Standalone exec does not inherit the desktop's selected-plugin context; + # feed the installed Browser skill through the native instructions hook. + "model_instructions_file": browser_runtime["skill"], + }) return configuration @@ -463,65 +603,80 @@ async def _run_codex(scenario_id: str, settings: ModelSettings, executable: str, from scripts.evaluation.__main__ import _provenance world = EvaluationWorld(scenario_id) - if world.scenario.kind == "browser" and not world.browser_url: - # CLI 当前没有浏览器工具；探针仍需有一个不真实连接的公开地址可渲染输入。 - world.configure_browser_url("http://127.0.0.1:1/fixture") - provenance = _provenance(world) - started = time.monotonic() - result: dict[str, Any] = {} - failure = None - with tempfile.TemporaryDirectory(prefix="moviepilot-native-evaluation-") as directory: - control_dir = Path(directory) - work_dir = control_dir / "workspace" - work_dir.mkdir() - catalog_result = await _execute([executable, "debug", "models", "--bundled"], "", _worker_environment(), work_dir, 10) - if catalog_result.get("error_type") or catalog_result.get("returncode") != 0: - raise RuntimeError("无法读取当前原生客户端自带模型目录") - catalog, catalog_metadata = _catalog(json.loads(catalog_result["stdout"]), settings.model) - (control_dir / "models.json").write_text(json.dumps(catalog), encoding="utf-8") - extra_native_tools = NATIVE_SHELL_TOOLS if scenario_id in { - "command_execution", "terminal_session", "terminal_pty_session", - } else frozenset() - async with EvaluationMcpServer(world) as server, EvaluationModelProxy( - settings, probe_only=probe_only, extra_native_tools=extra_native_tools, - ) as proxy: - config = _configuration(settings, proxy, server, control_dir, scenario_id=scenario_id) - use_app_server = scenario_id in NATIVE_TERMINAL_SCENARIOS - command = ( - [executable, "app-server", "--listen", "stdio://", "--strict-config"] - if use_app_server - else [executable, "exec", "--ignore-user-config", "--ignore-rules", "--strict-config", "--ephemeral", - "--skip-git-repo-check", "--sandbox", "read-only", "--json", "--color", "never", "-C", str(work_dir)] + fixture_context = _native_browser_fixture() if world.scenario.kind == "browser" else nullcontext(None) + with fixture_context as fixture: + if fixture is not None: + world.configure_browser_url(fixture[0]) + provenance = _provenance(world) + started = time.monotonic() + result: dict[str, Any] = {} + failure = None + with tempfile.TemporaryDirectory(prefix="moviepilot-native-evaluation-") as directory: + control_dir = Path(directory) + work_dir = control_dir / "workspace" + work_dir.mkdir() + catalog_result = await _execute([executable, "debug", "models", "--bundled"], "", _worker_environment(), work_dir, 10) + if catalog_result.get("error_type") or catalog_result.get("returncode") != 0: + raise RuntimeError("无法读取当前原生客户端自带模型目录") + catalog, catalog_metadata = _catalog(json.loads(catalog_result["stdout"]), settings.model) + (control_dir / "models.json").write_text(json.dumps(catalog), encoding="utf-8") + browser_runtime = ( + _browser_runtime_configuration(executable, control_dir) + if scenario_id in NATIVE_BROWSER_SCENARIOS else None ) - for key, value in config.items(): - command.extend(["-c", f"{key}={_toml(value)}"]) - environment = _worker_environment() - environment.update(MOVIEPILOT_EVAL_MODEL_TOKEN=proxy.bearer_token, - MOVIEPILOT_EVAL_MCP_TOKEN=server.bearer_token) - if use_app_server: - # app-server 必须使用临时状态根，避免读取或写入调用方的桌面会话和插件配置。 - environment["CODEX_HOME"] = str(control_dir) - else: - command.append("-") - try: - result = await ( - _execute_app_server( - command, world.model_input(), environment, work_dir, settings.timeout_seconds, - model=settings.model, reasoning_effort=settings.reasoning_effort, - ) - if use_app_server - else _execute(command, world.model_input(), environment, work_dir, settings.timeout_seconds) + extra_native_tools = NATIVE_SHELL_TOOLS if scenario_id in { + "command_execution", "terminal_session", "terminal_pty_session", + } else frozenset() + if browser_runtime is not None: + extra_native_tools |= NATIVE_BROWSER_TOOLS + async with EvaluationMcpServer(world) as server, EvaluationModelProxy( + settings, probe_only=probe_only, extra_native_tools=extra_native_tools, + ) as proxy: + config = _configuration( + settings, proxy, server, control_dir, scenario_id=scenario_id, browser_runtime=browser_runtime, ) - failure = result.get("error_type") - except (RuntimeError, TimeoutError, OSError) as error: - failure = type(error).__name__ - local_tokens = tuple( - value for value in (proxy.bearer_token, server.bearer_token, settings.account_id) - if isinstance(value, str) and value - ) - usage = proxy.snapshot() - server_stats = server.stats - skill_sha256 = server.skill_sha256 + use_app_server = scenario_id in NATIVE_TERMINAL_SCENARIOS + command = ( + [executable, "app-server", "--listen", "stdio://", "--strict-config"] + if use_app_server + else [executable, "exec", "--ignore-user-config", "--ignore-rules", "--strict-config", "--ephemeral", + "--skip-git-repo-check", "--sandbox", "read-only", "--json", "--color", "never", "-C", str(work_dir)] + ) + for key, value in config.items(): + command.extend(["-c", f"{key}={_toml(value)}"]) + environment = _worker_environment() + environment.update(MOVIEPILOT_EVAL_MODEL_TOKEN=proxy.bearer_token, + MOVIEPILOT_EVAL_MCP_TOKEN=server.bearer_token) + if use_app_server: + # app-server 必须使用临时状态根，避免读取或写入调用方的桌面会话和插件配置。 + environment["CODEX_HOME"] = str(control_dir) + else: + command.append("-") + try: + result = await ( + _execute_app_server( + command, world.model_input(), environment, work_dir, settings.timeout_seconds, + model=settings.model, reasoning_effort=settings.reasoning_effort, + ) + if use_app_server + else _execute(command, world.model_input(), environment, work_dir, settings.timeout_seconds) + ) + failure = result.get("error_type") + except (RuntimeError, TimeoutError, OSError) as error: + failure = type(error).__name__ + local_tokens = tuple( + value for value in (proxy.bearer_token, server.bearer_token, settings.account_id) + if isinstance(value, str) and value + ) + usage = proxy.snapshot() + server_stats = server.stats + skill_sha256 = server.skill_sha256 + if fixture is not None and fixture[1].get("clicked"): + # The native browser plugin bypasses the evaluation MCP server; the + # loopback callback is the independent proof that the button click ran. + world.record_browser("native_browser", { + "success": True, "execution_outcome": "succeeded", "rendered": "BROWSER_OK", + }) events, final_text, completed = _events(result.get("stdout", "")) if scenario_id in {"command_execution", "terminal_session", "terminal_pty_session"}: _record_native_command_events(world, events) @@ -544,7 +699,12 @@ async def _run_codex(scenario_id: str, settings: ModelSettings, executable: str, "native_stderr_sha256": hashlib.sha256(stderr.encode()).hexdigest(), "final_text": final_text, "final_report": final_report, "ledger": world.ledger, "fixture_server": server_stats, "exposed_skill_sha256": skill_sha256, - "tool_catalog_scope": "native_control_tools_and_shared_fixture_mcp", + "tool_catalog_scope": ( + "native_control_tools_shared_fixture_mcp_and_browser_plugin" + if scenario_id in NATIVE_BROWSER_SCENARIOS and browser_runtime is not None + else "native_control_tools_and_shared_fixture_mcp" + ), + "browser_runtime_enabled": bool(scenario_id in NATIVE_BROWSER_SCENARIOS and browser_runtime is not None), "native_transport": "app_server" if scenario_id in NATIVE_TERMINAL_SCENARIOS else "exec", } report["task_passed"] = report["passed"] diff --git a/scripts/evaluation/proxy.py b/scripts/evaluation/proxy.py index de1aecb12..2fc3d6926 100644 --- a/scripts/evaluation/proxy.py +++ b/scripts/evaluation/proxy.py @@ -35,17 +35,32 @@ NATIVE_COLLABORATION_TOOLS = frozenset({ }) NATIVE_MULTI_AGENT_TOOLS = NATIVE_CONTROL_TOOLS | NATIVE_COLLABORATION_TOOLS NATIVE_SHELL_TOOLS = frozenset({"exec_command", "write_stdin"}) -KNOWN_NAMESPACES = frozenset({"functions", "clock", "collaboration", "mcp__evaluation", "multi_agent_v1"}) +NATIVE_BROWSER_TOOLS = frozenset({ + "mcp__node_repl__js", "mcp__node_repl__js_reset", "mcp__node_repl__js_add_node_module_dir", + "mcp__node_repl__turn_ended", +}) +KNOWN_NAMESPACES = frozenset({ + "functions", "clock", "collaboration", "mcp__evaluation", "mcp__node_repl", "multi_agent_v1", +}) FIXTURE_TOOLS = frozenset({"moviepilot_api", "read_skill", "read_tool_result"}) +def _allowed_mcp_tool(normalized: str, extra_native_tools: frozenset[str]) -> bool: + """只放行评测夹具或当前浏览器场景明确授予的原生 MCP 工具。""" + if normalized == "mcp__evaluation" or normalized.startswith("mcp__evaluation__"): + return normalized == "mcp__evaluation" or normalized.removeprefix("mcp__evaluation__") in FIXTURE_TOOLS + if normalized == "mcp__node_repl": + return bool(NATIVE_BROWSER_TOOLS & extra_native_tools) + return normalized in extra_native_tools + + def allowed_tool(name: str, extra_native_tools: frozenset[str] = frozenset()) -> bool: """只允许原生控制、本地固定 MCP，以及当前场景明确授予的能力。""" normalized = name.removeprefix("functions.").replace(".", "__") return (normalized in NATIVE_CONTROL_TOOLS or normalized in extra_native_tools or normalized in {f"collaboration__{tool}" for tool in NATIVE_COLLABORATION_TOOLS} - or normalized in {f"mcp__evaluation__{tool}" for tool in FIXTURE_TOOLS} + or _allowed_mcp_tool(normalized, extra_native_tools) or (normalized.startswith("multi_agent_v1__") and normalized.removeprefix("multi_agent_v1__") in NATIVE_MULTI_AGENT_TOOLS)) @@ -69,8 +84,8 @@ def project_tools( raise ValueError("工具名称无效") qualified = f"{prefix}.{name}" if prefix else name normalized = qualified.removeprefix("functions.").replace(".", "__") - if tool["type"] == "mcp" or (normalized.startswith("mcp__") and not ( - normalized == "mcp__evaluation" or normalized.startswith("mcp__evaluation__") + if tool["type"] == "mcp" or (normalized.startswith("mcp__") and not _allowed_mcp_tool( + normalized, extra_native_tools )): raise ValueError(f"评测目录包含外部 MCP：{qualified}") if tool["type"] == "namespace": @@ -87,6 +102,8 @@ def project_tools( description = tool.get("description") sources = re.findall(r"(?m)^- ([^:\n]+):", description) if isinstance(description, str) else [] allowed_sources = {"Multi-agent tools", "evaluation"} + if NATIVE_BROWSER_TOOLS & extra_native_tools: + allowed_sources.add("node_repl") if (prefix or tool.get("execution") != "client" or not sources or "evaluation" not in sources or any(source not in allowed_sources for source in sources)): raise ValueError("原生工具搜索不是固定评测目录") diff --git a/tests/test_agent_evaluation_codex.py b/tests/test_agent_evaluation_codex.py index 18abb7222..26f609b79 100644 --- a/tests/test_agent_evaluation_codex.py +++ b/tests/test_agent_evaluation_codex.py @@ -7,6 +7,7 @@ import shlex import subprocess import sys import tomllib +import urllib.request from copy import deepcopy from pathlib import Path from types import SimpleNamespace @@ -272,6 +273,46 @@ def test_native_app_server_stream_timeout_follows_evaluation_budget() -> None: assert configuration["model_providers.evaluation"]["stream_idle_timeout_ms"] == 180_000 +def test_browser_runtime_expands_portable_skill_root_in_private_instruction_copy(tmp_path: Path, + monkeypatch: pytest.MonkeyPatch) -> None: + """独立 CLI 没有桌面插件上下文时，浏览器 Skill 的根路径仍必须可执行且不改写安装文件。""" + codex_home = tmp_path / "codex-home" + plugin_root = codex_home / ".tmp" / "bundled-marketplaces" / "openai-bundled" / "plugins" / "browser" + node_repl = tmp_path / "node" / "node_repl" + node_repl.parent.mkdir(parents=True) + node_repl.write_text("", encoding="utf-8") + (node_repl.parent / "node").write_text("", encoding="utf-8") + (node_repl.parent.parent / "lib" / "node_modules").mkdir(parents=True) + (plugin_root / "scripts").mkdir(parents=True) + (plugin_root / "scripts" / "browser-client.mjs").write_text("client", encoding="utf-8") + skill = plugin_root / "skills" / "control-in-app-browser" / "SKILL.md" + skill.parent.mkdir(parents=True) + skill.write_text("import('/scripts/browser-client.mjs')", encoding="utf-8") + service = codex_home / "plugins" / "cache" / "openai-bundled" / "browser" / "26.903.71938" / "scripts" / "browser-service.mjs" + service.parent.mkdir(parents=True) + service.write_text("service", encoding="utf-8") + instruction_dir = tmp_path / "control" + instruction_dir.mkdir() + monkeypatch.setenv("CODEX_HOME", str(codex_home)) + monkeypatch.setenv("MOVIEPILOT_EVAL_NODE_REPL", str(node_repl)) + runtime = codex._browser_runtime_configuration("/bin/codex", instruction_dir) + assert runtime is not None + expanded = Path(runtime["skill"]) + assert expanded != skill and expanded.is_file() + assert str(plugin_root) in expanded.read_text(encoding="utf-8") + assert "" not in expanded.read_text(encoding="utf-8") + assert runtime["node_repl"]["env"]["NODE_REPL_TRUSTED_SERVICES"] + + +def test_native_browser_fixture_records_only_loopback_click_callback() -> None: + """原生浏览器插件绕过评测 MCP 时，回环页面回调仍能作为独立点击证据。""" + with codex._native_browser_fixture() as (url, state): + page = urllib.request.urlopen(url, timeout=2).read().decode("utf-8") + assert "BROWSER_OK" in page and state["clicked"] is False + urllib.request.urlopen(url.replace("/fixture", "/clicked"), timeout=2).read() + assert state["clicked"] is True + + @pytest.mark.parametrize("catalog", [{}, {"models": []}, {"models": [{"slug": "gpt-test"}, {"slug": "gpt-test"}]}]) def test_missing_or_ambiguous_selected_model_never_falls_back(catalog: dict[str, Any]) -> None: """模型元数据缺失或重复时必须中止，不能换另一模型制造成功对照。"""