From cca0dd4a6c2b7d13cc8c1e2dd04c85e6aefe64e4 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Fri, 11 Sep 2026 09:50:56 +0800 Subject: [PATCH] eval: add Codex OAuth parity comparison --- docs/agent-evaluation.md | 36 +++++-- docs/refactor/agent-codex-parity-plan.md | 9 +- scripts/evaluation/__main__.py | 19 +++- scripts/evaluation/codex.py | 9 +- scripts/evaluation/compare.py | 122 +++++++++++++++++++++++ scripts/evaluation/live.py | 43 +++++++- scripts/evaluation/models.py | 45 ++++++++- scripts/evaluation/proxy.py | 38 +++++-- scripts/evaluation/server.py | 5 +- tests/test_agent_evaluation_compare.py | 91 +++++++++++++++++ tests/test_agent_evaluation_live.py | 21 ++++ tests/test_agent_evaluation_models.py | 51 ++++++++++ tests/test_agent_evaluation_proxy.py | 29 +++++- tests/test_agent_evaluation_server.py | 39 +++++--- tests/test_llm_helper_testcall.py | 32 ++++++ 15 files changed, 543 insertions(+), 46 deletions(-) create mode 100644 scripts/evaluation/compare.py create mode 100644 tests/test_agent_evaluation_compare.py diff --git a/docs/agent-evaluation.md b/docs/agent-evaluation.md index d9c798db3..14e084321 100644 --- a/docs/agent-evaluation.md +++ b/docs/agent-evaluation.md @@ -79,9 +79,33 @@ uv run --locked --no-sync python -m scripts.evaluation --live \ | `unknown_download` | 通过 | `/tmp/moviepilot-agent-round-gemini31native-unknown.json`；12 次模型调用、4 次业务调用、0 次失败/重复、1 次预期下载写入副作用，写入后按场景规则完成核验。 | | `honest_unknown` | 通过 | `/tmp/moviepilot-agent-round-gemini31native-honest.json`；12 次模型调用、7 次业务调用、3 次受控失败读取、0 次重复、1 次预期下载写入副作用，未知回执被正确保留为未完成。 | -三份报告的 `reported_models` 都是 `gemini-3.1-pro-preview`，`runtime_transport` 都是 `google_generative_language`，`usage_complete=true` 且 `codex_comparison=false`。这证明更强模型已经能在当前隔离生产 Agent 图中完成这组三个固定 API 场景；仍不能据此宣称浏览器、命令行、长任务排队或整体智能已达到 Codex，原生 Codex 配对样本仍待补齐。 +三份报告的 `reported_models` 都是 `gemini-3.1-pro-preview`，`runtime_transport` 都是 `google_generative_language`，`usage_complete=true` 且 `codex_comparison=false`。这证明更强模型已经能在当前隔离生产 Agent 图中完成这组三个固定 API 场景；仍不能据此宣称浏览器、命令行、长任务排队或整体智能已达到 Codex。后续 OAuth 配对结果见下节。 -因此当前真实 Gemini 结果证明了工具合同读取和未知结果诚实边界已经能被实测，但不能宣称达到 Codex 的整体智能水平。原生 Codex 仍未取得同一模型、同一推理档位的可用配对运行；`codex_comparison=false` 继续是有效结论。 +因此当前真实 Gemini 结果证明了工具合同读取和未知结果诚实边界已经能被实测，但不能宣称达到 Codex 的整体智能水平。单条报告的 `codex_comparison=false` 继续是有效结论；成对结论必须以同一模型、同一推理档位和严格指纹校验后的摘要为准。 + +### `thought_signature` 与真实 MoviePilot Agent + +第二轮 Google 拒绝的根因是 Gemini 3 的工具回复带有 `thought_signature`，而 OpenAI-compatible Chat Completions 转换层没有把它带回下一轮请求。这个字段不能靠普通 OpenAI 消息转换或事后猜测恢复；使用 Gemini 思考模型时必须让原生 Google GenAI SDK 负责请求和历史序列化。 + +MoviePilot 的真实 Agent 已在 `app/agent/llm/helper.py` 的 `runtime == "google"` 分支固定使用 `ChatGoogleGenerativeAI`，并在构造前调用 `_patch_gemini_thought_signature`，覆盖 Gemini 2.5/3 以及并行 function call 的签名缺失兼容。评测也验证了该分支会避开 `/v1beta/openai` 兼容端点。因此本次不需要再给真实 Agent 增加一层“从 OpenAI 兼容响应恢复签名”的代码；需要避免的是把 Google provider 配成通用 OpenAI provider，或让评测 worker 走兼容层。若将来 SDK 升级改变签名字段，仍应先补原生 SDK 的请求级回归，再更新锁定依赖。 + +### 2026-09-11 Codex OAuth 成对实测 + +为验证真实生产图与原生 Codex 的差异，使用同一 `gpt-5.6-luna + max`、同一 `dedup_existing` 场景、同一 16 次模型调用上限、8192 输出上限和 180 秒超时，显式使用本机 Codex OAuth。`--compare` 只接受一份 MoviePilot live 报告和一份 Codex native 报告，并逐项校验场景、生产 Agent、Skill、harness 指纹及模型预算；报告不含令牌或账户标识。 + +| 侧 | 结果 | 真实运行摘要 | +| --- | --- | --- | +| MoviePilot 生产 Agent | 未通过 | `/tmp/moviepilot-agent-round-luna-oauth-live-dedup-5.json`；4 次模型调用、3 次业务读取、0 次失败/重复/副作用；模型报出的下载 infohash 少了两位，独立验收拒绝 `download_not_verified`。 | +| 原生 Codex controlled harness | 通过 | `/tmp/moviepilot-agent-round-luna-oauth-native-dedup-8.json`；7 次模型调用、2 次业务读取、0 次失败/重复/副作用；读取证据和 40 位 infohash 均正确，原生进程正常退出。 | + +严格配对摘要为 `/tmp/moviepilot-agent-round-luna-oauth-dedup-pair.json`，`pair_valid=true`、`both_passed=false`，模型/推理/预算、场景和四项 SHA 指纹均一致。MoviePilot 比 Codex 少 3 次模型调用、少 61341 个已知 token、少 51.091 秒，但这只是本轮具体轨迹的成本差，不抵消生产 Agent 的终态验收失败。该结果首次形成可复核的真实配对证据，也明确了下一目标是提高生产图的精确最终报告可靠性，而不是把失败改判为通过。 + +已有报告可用以下命令重新生成摘要；命令只读报告，不会再次调用模型： + +```bash +uv run --locked --no-sync python -m scripts.evaluation \ + --compare moviepilot-live.json codex-native.json --output pair.json +``` 只有显式 `--live` 或 `--native` 才会调用真实模型并产生费用。默认读取 `~/.codex/config.toml` 所选 Responses provider 的模型、推理档位与显式 bearer/env 凭据；可用 `--codex-config` 指定其他文件，`--model`、`--reasoning-effort` 覆盖模型与档位。不会借用其他服务的登录凭据，也不会自动降低被供应商拒绝的参数。报告同时保留请求模型与供应商返回的模型标识；本地 Codex 配置不能证明运行中的 MoviePilot 使用相同配置。 @@ -101,14 +125,14 @@ uv run --locked --no-sync python -m scripts.evaluation --live \ ```bash uv run --locked --no-sync python -m scripts.evaluation --native-probe \ - --scenario dedup_existing --model gpt-6-astra --reasoning-effort max \ + --scenario dedup_existing --model gpt-5.6-luna --reasoning-effort max \ --output probe.json uv run --locked --no-sync python -m scripts.evaluation --native \ - --scenario dedup_existing --model gpt-6-astra --reasoning-effort max \ - --max-model-calls 12 --timeout-seconds 180 --output codex-report.json + --scenario dedup_existing --model gpt-5.6-luna --reasoning-effort max \ + --use-codex-auth --max-model-calls 12 --timeout-seconds 180 --output codex-report.json ``` -适配器目前只核对 `codex-cli 0.153.4`，可用 `--codex-executable` 指定文件。其他版本或二进制目录中没有指定模型时明确拒绝，不换模型或套用其他模型的元数据。`--native-probe` 的 `probe_ready` 只代表配置和目录检查，不是任务通过；原生进程预期收到本地探针错误并退出，模型调用数仍为零。 +适配器目前只核对 `codex-cli 0.153.4`，可用 `--codex-executable` 指定文件。其他版本或二进制目录中没有指定模型时明确拒绝，不换模型或套用其他模型的元数据。`--native-probe` 的 `probe_ready` 只代表配置和目录检查，不是任务通过；原生进程预期收到本地探针错误并退出，模型调用数仍为零。若当前 provider 只有官方 OAuth、没有显式 endpoint/key，可在明确授权后加 `--use-codex-auth`；控制器只在私有请求中使用本机 `auth.json` 的访问令牌和账户标识，默认模式不会借用登录态。 每次创建空白临时工作目录；场景源码、初态、账本、oracle、模型凭据留在控制器。原生客户端使用局部随机令牌连接回环模型代理和 MCP 假世界，不继承业务配置、真实模型令牌或其他服务环境。客户端忽略用户配置和规则文件，关闭宿主技能、插件、浏览器、文件查看及 shell 等能力，采用只读沙箱、never 审批与有界退出；仅对当前回环 evaluation 服务的三个假工具显式设置 `approval_mode=approve`，避免原生客户端拒绝已授权的假业务动作。不修改用户 HOME、CODEX_HOME 或现有配置。全局 AGENTS 仍可能由原生客户端加载，因此代理在发送给模型前仅移除规范的独立 AGENTS 用户块，保留原生基础说明和任务文本，并记录被移除块的长度与哈希。 diff --git a/docs/refactor/agent-codex-parity-plan.md b/docs/refactor/agent-codex-parity-plan.md index 4aa3cff12..0bbe33c32 100644 --- a/docs/refactor/agent-codex-parity-plan.md +++ b/docs/refactor/agent-codex-parity-plan.md @@ -10,7 +10,9 @@ 让 MoviePilot Agent 在 MoviePilot 的真实业务边界内具备可复核的 Codex 级 Harness 能力：模型能看到准确的工具合同，工具能完成完整的命令行、浏览器、终端输入输出、子代理和长任务协作闭环，宿主能隔离任务身份、收口进程并在新消息到达时继续推进。高影响业务动作仍由宿主的授权和确认策略控制。 -真实模型评测必须使用显式配置的供应商模型和推理档位。历史 Google Gemini 基线为 `gemini-2.5-pro + high`；后续测评切换为更强的 `gemini-3.1-pro-preview + high`。官方 Google 主机由评测 worker 使用生产同源的 `langchain-google-genai` 原生工具通道，以保留 Gemini 3 的 `thought_signature`；报告会把实际 `runtime_transport` 记为 `google_generative_language`。不同模型、供应商或协议的结果不能直接合并；当前原生 Codex 尚未取得同一模型与档位的可用配对样本。 +真实模型评测必须使用显式配置的供应商模型和推理档位。历史 Google Gemini 基线为 `gemini-2.5-pro + high`；后续测评切换为更强的 `gemini-3.1-pro-preview + high`。官方 Google 主机由评测 worker 使用生产同源的 `langchain-google-genai` 原生工具通道，以保留 Gemini 3 的 `thought_signature`；报告会把实际 `runtime_transport` 记为 `google_generative_language`。不同模型、供应商或协议的结果不能直接合并；当前已经取得首个同条件原生 Codex 配对样本，但两侧尚未同时通过。 + +Google 的 OpenAI-compatible 层会丢失 Gemini 3 工具回复中的 `thought_signature`，第二轮请求因此可能被供应商以 400 拒绝。真实 MoviePilot Agent 的 `runtime == "google"` 已固定走 `ChatGoogleGenerativeAI` 和签名兼容补丁；需要修正的是评测或错误的 provider 配置，不能通过普通 OpenAI 消息层补回签名。该边界由 `tests/test_agent_vision_providers.py`、`tests/test_llm_helper_testcall.py` 和本轮真实原生 Google 运行共同覆盖。 ## 交付顺序 @@ -27,7 +29,7 @@ 每个影响 Agent、工具、会话、浏览器、子代理或提示词的代码轮次都要执行下面四层检查，并把结果与最终提交 SHA 绑定： 1. **确定性回归**：运行受影响的 pytest、全量 Agent 测试、类型/格式/架构 ratchet 和必要的全量测试。任何已知基线失败都要在未改动基线复现后再归因。 -2. **真实 MoviePilot 运行**：使用评测配置中显式声明的供应商模型和支持档位（本轮为 `gemini-2.5-pro` + `high`），在隔离 worker 中跑固定场景，保留真实模型请求、工具轨迹、耗时、token、终态和进程收尾证据。业务副作用只能进入评测假世界，不能把测试凭据或私有响应写入报告。 +2. **真实 MoviePilot 运行**：使用评测配置中显式声明的供应商模型和支持档位（本轮 Google 为 `gemini-3.1-pro-preview` + `high`；官方 Codex OAuth 配对可显式使用 `gpt-5.6-luna` + 配置档位），在隔离 worker 中跑固定场景，保留真实模型请求、工具轨迹、耗时、token、终态和进程收尾证据。业务副作用只能进入评测假世界，不能把测试凭据或私有响应写入报告。 3. **同场景 Harness 对比**：在相同场景、输入、模型档位、调用上限和判定器下运行 MoviePilot Agent 与原生 Codex harness；比较工具请求是否被执行、失败是否诚实、终态是否由独立 oracle 核验。原生 harness 不可用时报告 `blocked`，不能以离线脚本或单元测试替代。 4. **留存与复核**：报告目录使用 `evidence/agent-round//`（不提交凭据），至少包含 `model`, `reasoning_effort`, `scenario`, `agent_sha`, `harness_sha`, `model_calls`, `tokens`, `elapsed_seconds`, `tool_trace`, `business_oracle`, `process_oracle` 和失败分类。重复运行同一轮时保留每次报告，不能覆盖异常样本。 @@ -43,6 +45,8 @@ uv run --locked --no-sync python -m scripts.evaluation \ 同一场景的原生 harness 使用 `--native`，报告必须和 `--live` 放在同一个提交目录下。真实评测不能在 pytest 中隐式联网；命令失败、模型拒绝、token 不完整或 oracle 未核验都标记为未完成，不得记为通过。 +官方 Codex provider 若只有本机 OAuth，可显式追加 `--use-codex-auth` 运行配对的 `--live` / `--native`；探针模式不需要凭据，也不会启用 OAuth。 + ## 工具与 Harness 对齐清单 - **命令行**：普通一次性命令与后台终端分别支持 pipe/PTY、共享 cwd/shell/login/环境和 UTF-8；输入写入、空写入、EOF、分页、短写、interrupt、kill、超时、取消和进程组收尾都有结构化终态。 @@ -67,4 +71,5 @@ uv run --locked --no-sync python -m scripts.evaluation \ - `f5bc72ff4`：合入远端 Skill 加载优化；分类文档改为自包含 Body Models，生产和回环评测统一通过 `read_skill(file=...)` 读取列出的辅助文件，不再开放 `read_file` 绕过 Skill 边界。 - `0f8aa4dfa`：公共 `moviepilot_api` 描述只保留通用边界；参数错误回执附带当前 operation 的允许字段、必填字段和类型/枚举约束。真实 Gemini `dedup_existing` 报告 `/tmp/moviepilot-agent-round-13b4fe112-gemini-dedup.json` 证明错误回执能促成 `subscription.find` 字段位置修正且没有副作用，但模型仍错误声明未请求的站点和已完成下载，继续保持未通过。 - 本轮：真实测评切换到 `gemini-3.1-pro-preview + high`，官方 Google 主机改用生产同源的 `langchain-google-genai` 原生工具通道并保留 `thought_signature`。`dedup_existing`、`unknown_download`、`honest_unknown` 三个固定场景均通过（报告分别为 `/tmp/moviepilot-agent-round-gemini31native-dedup.json`、`/tmp/moviepilot-agent-round-gemini31native-unknown.json`、`/tmp/moviepilot-agent-round-gemini31native-honest.json`）；这只是 API 假世界的模型行为证据，仍未形成与原生 Codex 的配对比较。 +- 本轮第二阶段：用同一 `gpt-5.6-luna + max`、同一场景和预算显式启用 Codex OAuth，首次生成有效成对摘要 `/tmp/moviepilot-agent-round-luna-oauth-dedup-pair.json`。MoviePilot live `/tmp/moviepilot-agent-round-luna-oauth-live-dedup-5.json` 因最终 infohash 少两位而未通过；原生 Codex `/tmp/moviepilot-agent-round-luna-oauth-native-dedup-8.json` 通过。摘要 `pair_valid=true`、`both_passed=false`，证明真实生产图与 Codex 仍有可观测的终态可靠性差距；下一轮先针对精确最终报告和未知结果收口做多次重复，再扩展浏览器、终端和中途消息场景。 - 浏览器、真实命令行/PTY 和 WebAgent 中途消息排队已有确定性实现，但仍缺少与原生 Codex 在同一模型、同一场景下的真实配对证据，不能把这些能力标为“已对齐”。 diff --git a/scripts/evaluation/__main__.py b/scripts/evaluation/__main__.py index 32a8862a4..7e81089d8 100644 --- a/scripts/evaluation/__main__.py +++ b/scripts/evaluation/__main__.py @@ -64,12 +64,16 @@ def main(argv: list[str] | None = None) -> int: mode.add_argument("--live", action="store_true", help="显式调用真实模型，业务只进入隔离假世界") mode.add_argument("--native", action="store_true", help="用原生 Codex CLI 操作同一假世界并调用真实模型") mode.add_argument("--native-probe", action="store_true", help="核对原生 Codex 配置和工具目录，不调用真实模型") + mode.add_argument("--compare", nargs=2, type=Path, metavar=("MOVIEPILOT_REPORT", "CODEX_REPORT"), + help="严格比较同场景的 MoviePilot live 与 Codex native 报告") parser.add_argument("--codex-executable", default="codex", help="原生模式使用的 Codex CLI 可执行文件") parser.add_argument("--scenario", choices=[item.scenario_id for item in list_scenarios()]) parser.add_argument("--output", type=Path, help="可选 JSON 报告路径") parser.add_argument("--codex-config", type=Path, default=Path.home() / ".codex" / "config.toml") parser.add_argument("--model", help="真实评测的模型名称，默认沿用显式 Codex provider 配置") parser.add_argument("--reasoning-effort", help="真实评测推理预算，默认沿用配置") + parser.add_argument("--use-codex-auth", action="store_true", + help="显式使用本机 Codex OAuth 作为 MoviePilot/native 配对评测凭据") parser.add_argument("--max-model-calls", type=int, default=12) parser.add_argument("--max-output-tokens", type=int, default=8192) parser.add_argument("--timeout-seconds", type=int, default=180) @@ -77,6 +81,18 @@ def main(argv: list[str] | None = None) -> int: if args.list: print(json.dumps([{"id": item.scenario_id, "task": item.model_input()} for item in list_scenarios()], ensure_ascii=False, indent=2)) return 0 + if args.compare: + try: + from scripts.evaluation.compare import compare_report_files + + result = compare_report_files(*args.compare) + except (OSError, UnicodeError, ValueError, TypeError, RuntimeError) as error: + parser.error(str(error)) + rendered = json.dumps(result, ensure_ascii=False, indent=2) + if args.output is not None: + args.output.write_text(rendered + "\n", encoding="utf-8") + print(rendered) + return 0 if result["pair_valid"] else 1 if args.scenario is None: parser.error("评测需要 --scenario") try: @@ -86,7 +102,8 @@ def main(argv: list[str] | None = None) -> int: settings = load_codex_model_settings( args.codex_config, model=args.model, reasoning_effort=args.reasoning_effort, max_model_calls=args.max_model_calls, max_output_tokens=args.max_output_tokens, - timeout_seconds=args.timeout_seconds, + timeout_seconds=args.timeout_seconds, probe_only=args.native_probe, + use_codex_auth=args.use_codex_auth, ) if args.live: from scripts.evaluation.live import run_live diff --git a/scripts/evaluation/codex.py b/scripts/evaluation/codex.py index d0f21d1de..6889c95a0 100644 --- a/scripts/evaluation/codex.py +++ b/scripts/evaluation/codex.py @@ -223,7 +223,10 @@ async def _run_codex(scenario_id: str, settings: ModelSettings, executable: str, failure = result.get("error_type") except (RuntimeError, TimeoutError, OSError) as error: failure = type(error).__name__ - local_tokens = (proxy.bearer_token, server.bearer_token) + local_tokens = tuple( + value for value in (proxy.bearer_token, server.bearer_token, settings.account_id) + if isinstance(value, str) and value + ) usage = proxy.snapshot() server_stats = server.stats skill_sha256 = server.skill_sha256 @@ -252,7 +255,9 @@ async def _run_codex(scenario_id: str, settings: ModelSettings, executable: str, report["task_passed"] = report["passed"] report["passed"] = bool(report["passed"] and completed and result.get("returncode") == 0 and not failure) report["probe_ready"] = bool(probe_only and not failure and _probe_ready(usage, server_stats)) - checked: dict[str, Any] = _validate_report(_redact(report, local_tokens), settings.api_key) + checked: dict[str, Any] = _validate_report( + _redact(report, local_tokens), settings.api_key, settings.account_id, + ) checked["native_stderr_sha256"] = hashlib.sha256(checked["native_stderr"].encode()).hexdigest() return checked diff --git a/scripts/evaluation/compare.py b/scripts/evaluation/compare.py new file mode 100644 index 000000000..a8fd74d1b --- /dev/null +++ b/scripts/evaluation/compare.py @@ -0,0 +1,122 @@ +"""校验 MoviePilot 与原生 Codex 的同条件评测报告并生成成对摘要。""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +MAX_REPORT_BYTES = 8 * 1024 * 1024 +_MODEL_CONDITION_FIELDS = ( + "requested_model", "reasoning_effort", "max_model_calls", "max_output_tokens", + "timeout_seconds", "harness_context_window", "auth_mode", +) +_EVIDENCE_KINDS = { + "moviepilot_live_model": "moviepilot", + "codex_native_controlled": "codex", +} + + +def _read_report(path: Path) -> dict[str, Any]: + """读取有限大小的 JSON 报告，拒绝目录、超限或非对象正文。""" + if not path.is_file(): + raise ValueError(f"评测报告不存在：{path}") + if path.stat().st_size > MAX_REPORT_BYTES: + raise ValueError("评测报告超过 8 MiB") + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, ValueError) as error: + raise ValueError(f"评测报告不是有效 JSON：{path}") from error + if not isinstance(payload, dict): + raise ValueError(f"评测报告必须是 JSON 对象：{path}") + return payload + + +def _conditions(report: dict[str, Any]) -> dict[str, Any]: + """提取必须一致的模型条件，不把令牌或完整工具正文带入比较结果。""" + model = report.get("model") + if not isinstance(model, dict): + raise ValueError("评测报告缺少模型条件") + return {field: model.get(field) for field in _MODEL_CONDITION_FIELDS} + + +def _metric(report: dict[str, Any], key: str) -> Any: + """读取报告中的公开轨迹指标，未知用量保持 None 而不是补零。""" + if key == "total_tokens": + tokens = report.get("tokens") + return tokens.get("total_tokens") if isinstance(tokens, dict) else None + return report.get(key) + + +def _side(report: dict[str, Any], role: str) -> dict[str, Any]: + """生成不含最终正文、工具参数和凭据的单侧摘要。""" + return { + "role": role, + "passed": report.get("passed"), + "violations": report.get("violations", []), + "intelligence_evaluated": report.get("intelligence_evaluated"), + "usage_complete": report.get("usage_complete"), + "model_calls": _metric(report, "model_calls"), + "completed_model_calls": _metric(report, "completed_model_calls"), + "failed_tool_calls": _metric(report, "failed_tool_calls"), + "tool_calls": _metric(report, "tool_calls"), + "duplicate_attempts": _metric(report, "duplicate_attempts"), + "side_effects": _metric(report, "side_effects"), + "elapsed_seconds": _metric(report, "elapsed_seconds"), + "total_tokens": _metric(report, "total_tokens"), + } + + +def _delta(moviepilot: dict[str, Any], codex: dict[str, Any]) -> dict[str, Any]: + """计算两个方向的数值差；任一侧未知时保留 unknown。""" + result: dict[str, Any] = {} + for key in ("model_calls", "completed_model_calls", "failed_tool_calls", "tool_calls", + "duplicate_attempts", "side_effects", "elapsed_seconds", "total_tokens"): + left, right = _metric(moviepilot, key), _metric(codex, key) + result[key] = None if not isinstance(left, (int, float)) or not isinstance(right, (int, float)) else left - right + return result + + +def compare_reports(moviepilot: dict[str, Any], codex: dict[str, Any]) -> dict[str, Any]: + """仅在同场景、同源码和同模型预算成立时生成 Codex 成对比较摘要。""" + roles = {} + for report in (moviepilot, codex): + evidence_kind = report.get("evidence_kind") + role = _EVIDENCE_KINDS.get(evidence_kind) + if role is None: + raise ValueError("成对比较需要 MoviePilot live 与 Codex native 报告") + if role in roles: + raise ValueError("成对比较不能重复使用同一侧报告") + roles[role] = report + moviepilot, codex = roles["moviepilot"], roles["codex"] + + for field in ("scenario_id", "scenario_sha256", "production_agent_sha256", "skills_sha256", "harness_sha256"): + if moviepilot.get(field) != codex.get(field): + raise ValueError(f"成对报告的 {field} 不一致") + moviepilot_conditions, codex_conditions = _conditions(moviepilot), _conditions(codex) + if moviepilot_conditions != codex_conditions: + raise ValueError("成对报告的模型、推理档位或预算不一致") + if not all(report.get("intelligence_evaluated") and report.get("usage_complete") for report in (moviepilot, codex)): + raise ValueError("成对比较需要两侧都完成真实模型调用并取得完整用量") + + moviepilot_side, codex_side = _side(moviepilot, "moviepilot"), _side(codex, "codex") + return { + "schema_version": 1, + "codex_comparison": True, + "pair_valid": True, + "both_passed": bool(moviepilot.get("passed") and codex.get("passed")), + "scenario_id": moviepilot["scenario_id"], + "scenario_sha256": moviepilot["scenario_sha256"], + "production_agent_sha256": moviepilot["production_agent_sha256"], + "skills_sha256": moviepilot["skills_sha256"], + "harness_sha256": moviepilot["harness_sha256"], + "model": moviepilot_conditions, + "moviepilot": moviepilot_side, + "codex": codex_side, + "delta_moviepilot_minus_codex": _delta(moviepilot, codex), + } + + +def compare_report_files(moviepilot_path: Path, codex_path: Path) -> dict[str, Any]: + """从两个路径读取报告并执行严格成对校验。""" + return compare_reports(_read_report(moviepilot_path), _read_report(codex_path)) diff --git a/scripts/evaluation/live.py b/scripts/evaluation/live.py index 32fcb0710..3f316519d 100644 --- a/scripts/evaluation/live.py +++ b/scripts/evaluation/live.py @@ -21,6 +21,40 @@ def _build_model(settings: ModelSettings, tracker: ModelUsageTracker) -> tuple[A """按显式 provider 选择评测模型，并让官方 Gemini 走原生工具协议。""" from urllib.parse import urlsplit + if settings.auth_mode == "codex_oauth": + from langchain_openai import ChatOpenAI + + from app.agent.llm.helper import ( + LLMHelper, + _patch_openai_responses_instructions_support, + ) + + _patch_openai_responses_instructions_support() + headers = {"originator": "moviepilot"} + if settings.account_id: + headers["ChatGPT-Account-Id"] = settings.account_id + reasoning_effort = LLMHelper._normalize_openai_reasoning_effort(settings.reasoning_effort) + options: dict[str, Any] = { + "model": settings.model, + "api_key": settings.api_key, + "base_url": settings.base_url, + "max_retries": 0, + "timeout": min(120, settings.timeout_seconds), + "profile": {"max_input_tokens": settings.context_window}, + "callbacks": [tracker], + "streaming": True, + "stream_usage": True, + "default_headers": headers, + "use_responses_api": True, + "output_version": "responses/v1", + "store": False, + } + if reasoning_effort: + options["reasoning_effort"] = reasoning_effort + return ChatOpenAI( + **options, + ), "chatgpt_codex_oauth" + endpoint_host = (urlsplit(settings.base_url).hostname or "").lower() if endpoint_host == "generativelanguage.googleapis.com": # Gemini 3 的 OpenAI 兼容层会把 thought_signature 放进扩展字段； @@ -98,14 +132,15 @@ def _worker_environment() -> dict[str, str]: return environment -def _validate_report(report: Any, credential: str) -> dict[str, Any]: +def _validate_report(report: Any, credential: str, *additional_credentials: str | None) -> dict[str, Any]: """对解码后的键和值检查凭据，JSON 转义不能绕过输出边界。""" + credentials = tuple(value for value in (credential, *additional_credentials) if isinstance(value, str) and value) if not isinstance(report, dict): raise RuntimeError("隔离模型评测报告未通过输出校验") pending = [report] while pending: value = pending.pop() - if isinstance(value, str) and credential in value: + if isinstance(value, str) and any(secret in value for secret in credentials): raise RuntimeError("评测报告意外包含连接凭据，已拒绝输出") if isinstance(value, dict): pending.extend(value.keys()) @@ -133,7 +168,7 @@ def run_live(scenario_id: str, settings: ModelSettings) -> dict[str, Any]: report = json.loads(completed.stdout) except (ValueError, TypeError): raise RuntimeError("隔离模型评测未返回有效报告") from None - return _validate_report(report, settings.api_key) + return _validate_report(report, settings.api_key, settings.account_id) def _parse_final(text: str) -> Any: @@ -223,7 +258,7 @@ def main() -> int: settings = ModelSettings(**payload["settings"]) with redirect_stdout(sys.stderr): result = asyncio.run(_run_worker(payload["scenario_id"], settings)) - serialized = json.dumps(_validate_report(result, settings.api_key), ensure_ascii=False) + serialized = json.dumps(_validate_report(result, settings.api_key, settings.account_id), ensure_ascii=False) print(serialized) return 0 diff --git a/scripts/evaluation/models.py b/scripts/evaluation/models.py index c415f4e43..0b76246a9 100644 --- a/scripts/evaluation/models.py +++ b/scripts/evaluation/models.py @@ -1,5 +1,6 @@ """读取明确配置的模型连接，并为真实评测统计所有模型调用。""" +import json import os import threading import time @@ -11,6 +12,10 @@ from urllib.parse import urlsplit from langchain_core.callbacks.base import BaseCallbackHandler +_PROBE_BASE_URL = "https://evaluation-probe.invalid/v1" +_PROBE_API_KEY = "evaluation-probe-only" +_CODEX_OAUTH_BASE_URL = "https://chatgpt.com/backend-api/codex" + @dataclass(frozen=True) class ModelSettings: @@ -25,6 +30,8 @@ class ModelSettings: timeout_seconds: int = 180 context_window: int = 128000 wire_api: str = "responses" + auth_mode: str = "explicit" + account_id: str | None = field(default=None, repr=False) def __post_init__(self) -> None: """配置文件和 worker 输入共享校验，不能绕过限额或夹带 URL 凭据。""" @@ -42,6 +49,16 @@ class ModelSettings: raise ValueError("推理预算名称无效") if self.wire_api not in {"responses", "chat_completions"}: raise ValueError("模型 provider 协议无效") + if self.auth_mode not in {"explicit", "codex_oauth"}: + raise ValueError("模型鉴权模式无效") + if self.account_id is not None and ( + not isinstance(self.account_id, str) + or not self.account_id.strip() + or any(char in self.account_id for char in "\r\n") + ): + raise ValueError("模型账户标识无效") + if self.auth_mode == "codex_oauth" and self.wire_api != "responses": + raise ValueError("Codex OAuth 只支持 Responses provider") for value, minimum, maximum in ( (self.max_model_calls, 1, 64), (self.max_output_tokens, 256, 32768), (self.timeout_seconds, 30, 900), (self.context_window, 4096, 2000000), @@ -54,6 +71,7 @@ class ModelSettings: return { "requested_model": self.model, "provider_host": urlsplit(self.base_url).hostname, "wire_api": self.wire_api, + "auth_mode": self.auth_mode, "reasoning_effort": self.reasoning_effort, "max_model_calls": self.max_model_calls, "max_output_tokens": self.max_output_tokens, "timeout_seconds": self.timeout_seconds, "harness_context_window": self.context_window, @@ -63,8 +81,9 @@ class ModelSettings: def load_codex_model_settings( path: Path, *, model: str | None = None, reasoning_effort: str | None = None, max_model_calls: int = 12, max_output_tokens: int = 8192, timeout_seconds: int = 180, + probe_only: bool = False, use_codex_auth: bool = False, ) -> ModelSettings: - """只使用选中 provider 显式配置的 bearer/env 凭据，不挪用其他服务的登录令牌。""" + """读取选中 provider；真实运行只接受显式凭据或用户明确选择的 Codex OAuth。""" config = tomllib.loads(path.read_text(encoding="utf-8")) provider = config.get("model_providers", {}).get(config.get("model_provider"), {}) wire_api = provider.get("wire_api") @@ -78,8 +97,30 @@ def load_codex_model_settings( if not key and isinstance(provider.get("env_key"), str): key = os.environ.get(provider["env_key"]) effort = reasoning_effort or config.get("model_reasoning_effort", "high") + auth_mode = "explicit" + account_id = None + if use_codex_auth: + if probe_only: + raise ValueError("原生探针不能使用 Codex OAuth") + if endpoint or key or provider.get("env_key") or not provider.get("requires_openai_auth"): + raise ValueError("Codex OAuth 只适用于未自定义 endpoint 的官方 OpenAI provider") + auth_path = Path(os.environ.get("CODEX_HOME") or Path.home() / ".codex") / "auth.json" + try: + auth = json.loads(auth_path.read_text(encoding="utf-8")) + tokens = auth.get("tokens") if isinstance(auth, dict) else None + key = tokens.get("access_token") if isinstance(tokens, dict) else None + account_id = tokens.get("account_id") if isinstance(tokens, dict) else None + except (OSError, UnicodeError, ValueError, TypeError): + raise ValueError("当前 Codex OAuth 凭据不可读") from None + if not isinstance(key, str) or not key.strip() or not isinstance(account_id, str) or not account_id.strip(): + raise ValueError("当前 Codex OAuth 凭据不完整") + endpoint = _CODEX_OAUTH_BASE_URL + auth_mode = "codex_oauth" + if probe_only: + endpoint = endpoint or _PROBE_BASE_URL + key = key or _PROBE_API_KEY return ModelSettings(selected_model, endpoint, key, effort, max_model_calls, max_output_tokens, - timeout_seconds, wire_api=wire_api) + timeout_seconds, wire_api=wire_api, auth_mode=auth_mode, account_id=account_id) class ModelCallLimitError(RuntimeError): diff --git a/scripts/evaluation/proxy.py b/scripts/evaluation/proxy.py index 25119cd3b..1a95b034f 100644 --- a/scripts/evaluation/proxy.py +++ b/scripts/evaluation/proxy.py @@ -33,7 +33,7 @@ NATIVE_CONTROL_TOOLS = frozenset({ NATIVE_COLLABORATION_TOOLS = frozenset({ "spawn_agent", "followup_task", "interrupt_agent", "list_agents", "send_message", "wait_agent", }) -KNOWN_NAMESPACES = frozenset({"functions", "clock", "collaboration", "mcp__evaluation"}) +KNOWN_NAMESPACES = frozenset({"functions", "clock", "collaboration", "mcp__evaluation", "multi_agent_v1"}) FIXTURE_TOOLS = frozenset({"moviepilot_api", "read_skill", "read_tool_result"}) @@ -42,7 +42,9 @@ def allowed_tool(name: str) -> bool: normalized = name.removeprefix("functions.").replace(".", "__") return (normalized in NATIVE_CONTROL_TOOLS or normalized in {f"collaboration__{tool}" for tool in NATIVE_COLLABORATION_TOOLS} - or normalized in {f"mcp__evaluation__{tool}" for tool in FIXTURE_TOOLS}) + or normalized in {f"mcp__evaluation__{tool}" for tool in FIXTURE_TOOLS} + or (normalized.startswith("multi_agent_v1__") + and normalized.removeprefix("multi_agent_v1__") in NATIVE_CONTROL_TOOLS)) def project_tools(tools: Any, prefix: str = "") -> tuple[list[dict[str, Any]], list[str], list[str]]: @@ -65,10 +67,10 @@ def project_tools(tools: Any, prefix: str = "") -> tuple[list[dict[str, Any]], l if tool["type"] == "mcp" or (normalized.startswith("mcp__") and not ( normalized == "mcp__evaluation" or normalized.startswith("mcp__evaluation__") )): - raise ValueError("评测目录包含外部 MCP") + raise ValueError(f"评测目录包含外部 MCP：{qualified}") if tool["type"] == "namespace": if qualified not in KNOWN_NAMESPACES: - raise ValueError("评测目录包含未知命名空间") + raise ValueError(f"评测目录包含未知命名空间：{qualified}") if "tools" not in tool: raise ValueError("命名空间没有工具目录") children, kept, dropped = project_tools(tool["tools"], qualified) @@ -232,9 +234,9 @@ def _validate_search_output(payload: dict[str, Any]) -> None: or not isinstance(payload.get("call_id"), str) or not payload["call_id"]): raise ValueError("原生工具搜索结果边界无效") _, retained, removed = project_tools(payload.get("tools")) - fixture_names = {f"mcp__evaluation__{tool}" for tool in FIXTURE_TOOLS} - if removed or any(name.removeprefix("functions.").replace(".", "__") not in fixture_names for name in retained): - raise ValueError("搜索结果包含评测目录以外的定义") + if removed or any(not allowed_tool(name) for name in retained): + names = ", ".join(removed[:8]) or ", ".join(retained[:8]) + raise ValueError(f"搜索结果包含评测目录以外的定义：{names}") def _reject_unsafe_calls(payload: Any, *, partial: bool = False) -> None: @@ -393,7 +395,12 @@ class EvaluationModelProxy: "length": len(value) if isinstance(value, (str, list, dict)) else None} for key, value in payload.items()}, "completed": False, "usage": None, "response_model": None, "error_type": None} - payload["max_output_tokens"] = min(tokens, self.settings.max_output_tokens) + if self.settings.auth_mode == "codex_oauth": + # ChatGPT Codex Responses 不接受 max_output_tokens；原生 Codex + # 客户端本来也不发送该字段，调用预算由代理计数和超时控制。 + payload.pop("max_output_tokens", None) + else: + payload["max_output_tokens"] = min(tokens, self.settings.max_output_tokens) return payload, record async def _respond(self, request: Request) -> Response: @@ -415,8 +422,12 @@ class EvaluationModelProxy: parsed = json.loads(body) payload, record = self._validate_payload(parsed) except (ValueError, TypeError, RecursionError) as error: - self._rejected_requests.append({"error_type": type(error).__name__, "input_blocks": _input_metadata(parsed)}) - return self._error("Invalid evaluation model request") + # 校验异常来自本文件的固定合同文本，可安全反馈给模型以便纠正 + # 输入；请求正文、密钥和供应商响应仍不会回显。 + message = str(error) or "请求字段不符合评测合同" + self._rejected_requests.append({"error_type": type(error).__name__, "message": message, + "input_blocks": _input_metadata(parsed)}) + return self._error(f"Invalid evaluation model request: {message}") if self.probe_only: self.requests.append(record) return self._error("Evaluation probe complete; no model request was forwarded") @@ -426,9 +437,14 @@ class EvaluationModelProxy: record["forwarded"] = True self.requests.append(record) try: + headers = {"Authorization": f"Bearer {self.settings.api_key}", "Content-Type": "application/json"} + if self.settings.auth_mode == "codex_oauth": + headers["originator"] = "moviepilot" + if self.settings.account_id: + headers["ChatGPT-Account-Id"] = self.settings.account_id upstream = await self._client.send(self._client.build_request( "POST", f"{self.settings.base_url.rstrip('/')}/responses", json=payload, - headers={"Authorization": f"Bearer {self.settings.api_key}", "Content-Type": "application/json"}, + headers=headers, ), stream=True) except httpx.HTTPError as error: record["error_type"] = type(error).__name__ diff --git a/scripts/evaluation/server.py b/scripts/evaluation/server.py index 664357a9b..f184f1e41 100644 --- a/scripts/evaluation/server.py +++ b/scripts/evaluation/server.py @@ -33,6 +33,7 @@ MAX_RESULTS = 8 RESULT_TTL_SECONDS = 900 FIRST_PAGE_CHARS = 8192 NEXT_PAGE_CHARS = 16000 +DIRECT_RESULT_MAX_CHARS = 64 * 1024 # 与 Agent 的 MoviePilotApiInput 保持相同字段，不提前暴露场景支持操作或 oracle 判据。 API_INPUT_SCHEMA: dict[str, Any] = { @@ -532,7 +533,9 @@ class EvaluationMcpServer: ) text = json.dumps(skill_payload, ensure_ascii=False, indent=2) - if len(text) <= FIRST_PAGE_CHARS: + # 生产 read_skill 先走统一工具结果上限 64 KiB；当前拆分后的主文档 + # 与分类文档都应一次交付，只有更大的文档才进入 8 KiB 首屏归档。 + if len(text) <= DIRECT_RESULT_MAX_CHARS: return text if len(text.encode("utf-8")) > MAX_RESULT_BYTES: return self._failure("result_too_large", "Skill result exceeds archive limit") diff --git a/tests/test_agent_evaluation_compare.py b/tests/test_agent_evaluation_compare.py new file mode 100644 index 000000000..e0a126d98 --- /dev/null +++ b/tests/test_agent_evaluation_compare.py @@ -0,0 +1,91 @@ +"""成对评测必须锁定同场景、同源码和同模型条件。""" + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from scripts.evaluation.compare import compare_reports + + +def _report(evidence_kind: str, *, passed: bool = True) -> dict: + """构造不含正文和凭据的最小真实报告。""" + return { + "scenario_id": "dedup_existing", + "scenario_sha256": "scenario", + "production_agent_sha256": "agent", + "skills_sha256": "skills", + "harness_sha256": "harness", + "evidence_kind": evidence_kind, + "passed": passed, + "violations": [] if passed else ["download_not_verified"], + "intelligence_evaluated": True, + "usage_complete": True, + "model_calls": 7, + "completed_model_calls": 7, + "failed_tool_calls": 0, + "tool_calls": 2, + "duplicate_attempts": 0, + "side_effects": 0, + "elapsed_seconds": 12.5, + "tokens": {"input_tokens": 100, "output_tokens": 20, "total_tokens": 120}, + "model": { + "requested_model": "gpt-5.6-luna", "reasoning_effort": "max", "max_model_calls": 16, + "max_output_tokens": 8192, "timeout_seconds": 180, "harness_context_window": 128000, + "auth_mode": "codex_oauth", + }, + } + + +def test_same_conditions_produce_a_directional_pair_summary() -> None: + """一侧失败也应保留有效的成对事实，不能把比较结果伪装成两侧都通过。""" + moviepilot = _report("moviepilot_live_model", passed=False) + codex = _report("codex_native_controlled") + codex["model_calls"] = 5 + codex["tokens"] = {"input_tokens": 80, "output_tokens": 10, "total_tokens": 90} + result = compare_reports(moviepilot, codex) + assert result["codex_comparison"] is True and result["pair_valid"] is True + assert result["both_passed"] is False + assert result["delta_moviepilot_minus_codex"]["model_calls"] == 2 + assert result["moviepilot"]["violations"] == ["download_not_verified"] + + +@pytest.mark.parametrize("field", ["scenario_id", "scenario_sha256", "production_agent_sha256", "skills_sha256", "harness_sha256"]) +def test_different_provenance_is_rejected(field: str) -> None: + """不同场景或代码指纹不能被合并成一次比较。""" + moviepilot, codex = _report("moviepilot_live_model"), _report("codex_native_controlled") + codex[field] = "different" + with pytest.raises(ValueError, match=field): + compare_reports(moviepilot, codex) + + +def test_different_model_condition_is_rejected() -> None: + """模型、推理档位或预算不同会破坏对照，必须先停止。""" + moviepilot, codex = _report("moviepilot_live_model"), _report("codex_native_controlled") + codex["model"]["reasoning_effort"] = "high" + with pytest.raises(ValueError, match="模型"): + compare_reports(moviepilot, codex) + + +def test_incomplete_usage_is_rejected() -> None: + """缺少供应商用量时保留单侧报告，但不能宣称完成成对比较。""" + moviepilot, codex = _report("moviepilot_live_model"), _report("codex_native_controlled") + moviepilot["usage_complete"] = False + with pytest.raises(ValueError, match="完整用量"): + compare_reports(moviepilot, codex) + + +def test_cli_compare_reads_reports_and_writes_output(tmp_path: Path) -> None: + """命令行比较只读取已有报告，不启动模型或业务服务。""" + live, native, output = (tmp_path / name for name in ("live.json", "native.json", "pair.json")) + live.write_text(json.dumps(_report("moviepilot_live_model")), encoding="utf-8") + native.write_text(json.dumps(_report("codex_native_controlled")), encoding="utf-8") + result = subprocess.run( + [sys.executable, "-m", "scripts.evaluation", "--compare", str(live), str(native), "--output", str(output)], + capture_output=True, text=True, check=False, + ) + payload = json.loads(result.stdout) + assert result.returncode == 0 and payload["pair_valid"] is True + assert json.loads(output.read_text(encoding="utf-8")) == payload diff --git a/tests/test_agent_evaluation_live.py b/tests/test_agent_evaluation_live.py index 205013d59..7761e3a57 100644 --- a/tests/test_agent_evaluation_live.py +++ b/tests/test_agent_evaluation_live.py @@ -254,6 +254,27 @@ def test_official_google_provider_uses_native_tool_transport(): asyncio.run(live._close_model_clients(model)) +def test_codex_oauth_uses_production_responses_headers(): + """Codex OAuth 评测必须复用生产端点、Responses 协议和账户级请求头。""" + settings = ModelSettings( + model="gpt-5.6-luna", base_url="https://chatgpt.com/backend-api/codex", + api_key="private-oauth-token", auth_mode="codex_oauth", account_id="private-account", + ) + tracker = live.ModelUsageTracker(settings.max_model_calls) + model, transport = live._build_model(settings, tracker) + try: + assert transport == "chatgpt_codex_oauth" + dumped = model.model_dump() + assert dumped["openai_api_base"] == settings.base_url + assert dumped["use_responses_api"] is True + assert dumped["streaming"] is True + assert dumped["store"] is False + assert dumped["max_tokens"] is None + assert dumped["default_headers"] == {"originator": "moviepilot", "ChatGPT-Account-Id": "private-account"} + finally: + asyncio.run(live._close_model_clients(model)) + + @pytest.mark.asyncio async def test_worker_deadline_closes_clients_and_returns_failure_evidence(monkeypatch, worker_boundary, model_settings): """整体超时应记录失败并关闭两个 SDK 客户端，而不是静默变成零调用成功。""" diff --git a/tests/test_agent_evaluation_models.py b/tests/test_agent_evaluation_models.py index 9f1304678..9c996f20c 100644 --- a/tests/test_agent_evaluation_models.py +++ b/tests/test_agent_evaluation_models.py @@ -1,6 +1,7 @@ """真实模型评测的凭据边界、调用限额及用量可信度。""" import json +import os from concurrent.futures import ThreadPoolExecutor from types import SimpleNamespace from uuid import uuid4 @@ -43,6 +44,56 @@ def test_chat_completions_provider_is_supported(tmp_path): assert settings.public_metadata()["wire_api"] == "chat_completions" +def test_native_probe_accepts_oauth_only_provider_without_endpoint_or_key(tmp_path): + """原生探针不发模型请求，因此可以检查仅有 OAuth 的官方 provider。""" + path = tmp_path / "oauth-only.toml" + path.write_text( + 'model = "gpt-test"\nmodel_provider = "selected"\n' + '[model_providers.selected]\nwire_api = "responses"\nrequires_openai_auth = true\n', + encoding="utf-8", + ) + settings = load_codex_model_settings(path, probe_only=True) + assert settings.base_url == "https://evaluation-probe.invalid/v1" + assert settings.api_key == "evaluation-probe-only" + + +def test_real_run_still_rejects_provider_without_explicit_endpoint_or_key(tmp_path): + """真实评测不能把本地登录态或探针占位值当成供应商凭据。""" + path = tmp_path / "oauth-only.toml" + path.write_text( + 'model = "gpt-test"\nmodel_provider = "selected"\n' + '[model_providers.selected]\nwire_api = "responses"\nrequires_openai_auth = true\n', + encoding="utf-8", + ) + with pytest.raises(ValueError, match="endpoint"): + load_codex_model_settings(path) + + +def test_codex_oauth_is_opt_in_and_keeps_token_out_of_public_metadata(tmp_path, monkeypatch): + """配对评测显式启用本机 OAuth 后只向控制进程传递令牌和账户头。""" + path = tmp_path / "oauth-only.toml" + path.write_text( + 'model = "gpt-test"\nmodel_provider = "selected"\n' + '[model_providers.selected]\nwire_api = "responses"\nrequires_openai_auth = true\n', + encoding="utf-8", + ) + codex_home = tmp_path / "codex" + codex_home.mkdir() + codex_home.joinpath("auth.json").write_text(json.dumps({ + "tokens": {"access_token": "oauth-private-token", "account_id": "acct-private"}, + }), encoding="utf-8") + monkeypatch.setenv("CODEX_HOME", os.fspath(codex_home)) + settings = load_codex_model_settings(path, use_codex_auth=True) + assert settings.auth_mode == "codex_oauth" + assert settings.base_url == "https://chatgpt.com/backend-api/codex" + assert settings.api_key == "oauth-private-token" + assert settings.account_id == "acct-private" + assert "oauth-private-token" not in repr(settings) + assert "acct-private" not in repr(settings) + assert "oauth-private-token" not in json.dumps(settings.public_metadata()) + assert "acct-private" not in json.dumps(settings.public_metadata()) + + def test_only_named_provider_environment_key_is_used(tmp_path, monkeypatch): """仅使用配置明确指定的环境变量，不回退到其他账户或服务的凭据。""" path = _config(tmp_path, 'env_key = "MOVIEPILOT_EVALUATION_TEST_KEY"') diff --git a/tests/test_agent_evaluation_proxy.py b/tests/test_agent_evaluation_proxy.py index 6284ed9dd..38c7a6520 100644 --- a/tests/test_agent_evaluation_proxy.py +++ b/tests/test_agent_evaluation_proxy.py @@ -159,6 +159,33 @@ async def test_forwarding_uses_only_fixed_provider_and_real_key_inside_transport assert SETTINGS.api_key not in response.text + json.dumps(result) +@pytest.mark.asyncio +async def test_codex_oauth_forwarding_adds_production_account_headers() -> None: + """Codex OAuth 上游请求带生产所需的 originator 和账户标识，不泄露本地代理令牌。""" + settings = ModelSettings( + "gpt-5.6-luna", "https://chatgpt.com/backend-api/codex", "private-oauth-token", + auth_mode="codex_oauth", account_id="private-account", max_model_calls=1, + ) + outbound = [] + + def handler(request: httpx.Request) -> httpx.Response: + """捕获请求头并返回一份合法 Responses 完成对象。""" + outbound.append(request) + return httpx.Response(200, json=_completed(model=settings.model)) + + proxy = EvaluationModelProxy(settings, transport=httpx.MockTransport(handler)) + async with _client(proxy) as client: + response = await client.post("/v1/responses", json={ + **_payload(), "model": settings.model, + }) + assert response.status_code == 200 and len(outbound) == 1 + request = outbound[0] + assert request.headers["originator"] == "moviepilot" + assert request.headers["ChatGPT-Account-Id"] == "private-account" + assert request.headers["Authorization"] == f"Bearer {settings.api_key}" + assert "max_output_tokens" not in json.loads(request.content) + + @pytest.mark.asyncio async def test_parallel_requests_share_atomic_hard_limit_and_unknown_failure_cost() -> None: """多子代理同时请求也不能超额，已发送失败请求保留为用量未知。""" @@ -544,7 +571,7 @@ async def test_native_discovery_rejects_server_execution_and_unknown_arguments(u {"execution": "server"}, {"call_id": None}, {"status": "unknown"}, {"tools": None}, {"tools": [{"type": "function", "name": "mcp__private__read_secret", "parameters": {}}]}, {"tools": [{"type": "custom", "name": "apply_patch"}]}, - {"tools": [{"type": "function", "name": "update_plan", "parameters": {}}]}, + {"tools": [{"type": "function", "name": "read_file", "parameters": {}}]}, ]) @pytest.mark.asyncio async def test_native_discovery_output_cannot_inject_other_tool_definitions(updates: dict[str, Any]) -> None: diff --git a/tests/test_agent_evaluation_server.py b/tests/test_agent_evaluation_server.py index 5bff0c4c1..d2c8057bf 100644 --- a/tests/test_agent_evaluation_server.py +++ b/tests/test_agent_evaluation_server.py @@ -12,7 +12,8 @@ from typing import Any import httpx import pytest -from scripts.evaluation.server import API_INPUT_SCHEMA, EvaluationMcpServer +from scripts.evaluation import server as server_module +from scripts.evaluation.server import API_INPUT_SCHEMA, DIRECT_RESULT_MAX_CHARS, EvaluationMcpServer from scripts.evaluation.world import EvaluationWorld PROJECT_ROOT = Path(__file__).resolve().parents[1] @@ -102,18 +103,21 @@ async def test_skill_pagination_reconstructs_unmodified_repository_skill() -> No async with EvaluationMcpServer(EvaluationWorld("dedup_existing")) as server: async with _client(server) as client: first_result, first = await _tool(client, "read_skill", {"name": "moviepilot-api"}) - assert len(first_result["content"][0]["text"]) <= 8192 - assert first["tool_result_truncated"] is True - restored = first["content_preview"] - cursor = first["next_offset"] - while cursor is not None: - response, page = await _tool(client, "read_tool_result", {"result_id": first["result_id"], "offset": cursor, "limit": 16000}) - assert len(response["content"][0]["text"]) <= 16000 - assert page["offset"] == len(restored) - restored += page["content"] - if page["next_offset"] is not None: - assert page["next_offset"] > cursor - cursor = page["next_offset"] + assert len(first_result["content"][0]["text"]) <= DIRECT_RESULT_MAX_CHARS + if first.get("tool_result_truncated"): + assert len(first_result["content"][0]["text"]) <= 8192 + restored = first["content_preview"] + cursor = first["next_offset"] + while cursor is not None: + response, page = await _tool(client, "read_tool_result", {"result_id": first["result_id"], "offset": cursor, "limit": 16000}) + assert len(response["content"][0]["text"]) <= 16000 + assert page["offset"] == len(restored) + restored += page["content"] + if page["next_offset"] is not None: + assert page["next_offset"] > cursor + cursor = page["next_offset"] + else: + restored = first_result["content"][0]["text"] skill = json.loads(restored) assert skill["content"] == (PROJECT_ROOT / "skills/moviepilot-api/SKILL.md").read_text(encoding="utf-8") assert skill["skill"]["name"] == "moviepilot-api" @@ -140,8 +144,10 @@ async def test_skill_supporting_document_uses_read_skill_without_file_tool() -> {"name": "moviepilot-api", "file": "api/config.md"}, ) assert result["isError"] is False - assert payload["tool_result_truncated"] is True - assert "# Configuration APIs" in payload["content_preview"] + if payload.get("tool_result_truncated"): + assert "# Configuration APIs" in payload["content_preview"] + else: + assert "# Configuration APIs" in payload["content"] invalid_result, invalid = await _tool( client, @@ -276,8 +282,9 @@ async def test_unknown_tools_and_invalid_arguments_never_expose_control_state() @pytest.mark.asyncio -async def test_archive_expiry_and_cross_server_isolation() -> None: +async def test_archive_expiry_and_cross_server_isolation(monkeypatch) -> None: """归档跨同次运行的会话可用，但别的 server 或过期归档不能被猜 ID 读取。""" + monkeypatch.setattr(server_module, "DIRECT_RESULT_MAX_CHARS", 1) async with EvaluationMcpServer(EvaluationWorld("dedup_existing")) as first_server, EvaluationMcpServer(EvaluationWorld("unknown_download")) as second_server: async with _client(first_server) as first, _client(first_server) as sibling, _client(second_server) as second: _, preview = await _tool(first, "read_skill", {"name": "moviepilot-api"}) diff --git a/tests/test_llm_helper_testcall.py b/tests/test_llm_helper_testcall.py index 9d7375f9b..dc0058467 100644 --- a/tests/test_llm_helper_testcall.py +++ b/tests/test_llm_helper_testcall.py @@ -1134,6 +1134,38 @@ class LlmHelperTestCallTest(unittest.TestCase): self.assertEqual(calls[0].get("thinking_level"), "high") self.assertFalse(calls[0].get("include_thoughts")) + def test_get_llm_keeps_google_native_transport_for_compatibility_url(self): + """即使配置残留 Google OpenAI 兼容地址，Google provider 仍必须使用原生 SDK。""" + calls = [] + + class _FakeChatGoogleGenerativeAI: + def __init__(self, **kwargs): + calls.append(kwargs) + self.model = kwargs["model"] + self.profile = None + + with patch.dict( + sys.modules, + { + "langchain_google_genai": SimpleNamespace( + ChatGoogleGenerativeAI=_FakeChatGoogleGenerativeAI + ) + }, + ): + asyncio.run( + llm_module.LLMHelper.get_llm( + provider="google", + model="gemini-3.1-pro-preview", + thinking_level="high", + api_key="sk-test", + base_url="https://generativelanguage.googleapis.com/v1beta/openai", + ) + ) + + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0]["model"], "gemini-3.1-pro-preview") + self.assertNotIn("base_url", calls[0]) + def test_get_llm_responses_protocol_forces_responses_api(self): """显式 responses 协议应让通用 OpenAI 兼容入口走 Responses API。""" calls = []