From 7ab8cddccb491d9ea8d8a069edb384cfe4dd0fa2 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Fri, 11 Sep 2026 06:26:26 +0800 Subject: [PATCH] feat(evaluation): support chat completions providers --- scripts/evaluation/codex.py | 2 ++ scripts/evaluation/live.py | 17 +++++++++++------ scripts/evaluation/models.py | 12 +++++++++--- tests/test_agent_evaluation_live.py | 18 ++++++++++++++++++ tests/test_agent_evaluation_models.py | 15 ++++++++++++++- 5 files changed, 54 insertions(+), 10 deletions(-) diff --git a/scripts/evaluation/codex.py b/scripts/evaluation/codex.py index 43fe99fc0..d0f21d1de 100644 --- a/scripts/evaluation/codex.py +++ b/scripts/evaluation/codex.py @@ -259,6 +259,8 @@ async def _run_codex(scenario_id: str, settings: ModelSettings, executable: str, def run_codex(scenario_id: str, settings: ModelSettings, *, executable: str = "codex", probe_only: bool = False) -> dict[str, Any]: """仅运行已核对的原生版本，探针不会向真实供应商发送请求。""" + if settings.wire_api != "responses": + raise RuntimeError("原生 Codex 对照目前要求 Responses provider；当前配置是 Chat Completions") binary = shutil.which(executable) if binary is None: raise RuntimeError("未找到原生 Codex CLI") diff --git a/scripts/evaluation/live.py b/scripts/evaluation/live.py index 6899943c7..100a7f3a2 100644 --- a/scripts/evaluation/live.py +++ b/scripts/evaluation/live.py @@ -97,12 +97,17 @@ async def _run_worker(scenario_id: str, settings: ModelSettings) -> dict[str, An from scripts.evaluation.world import EvaluationWorld tracker = ModelUsageTracker(settings.max_model_calls) - model = ChatOpenAI( - model=settings.model, api_key=settings.api_key, base_url=settings.base_url, - use_responses_api=True, reasoning={"effort": settings.reasoning_effort}, - max_tokens=settings.max_output_tokens, max_retries=0, timeout=min(120, settings.timeout_seconds), - profile={"max_input_tokens": settings.context_window}, callbacks=[tracker], - ) + model_options: dict[str, Any] = { + "model": settings.model, "api_key": settings.api_key, "base_url": settings.base_url, + "max_tokens": settings.max_output_tokens, "max_retries": 0, + "timeout": min(120, settings.timeout_seconds), + "profile": {"max_input_tokens": settings.context_window}, "callbacks": [tracker], + } + if settings.wire_api == "responses": + model_options.update(use_responses_api=True, reasoning={"effort": settings.reasoning_effort}) + else: + model_options.update(use_responses_api=False, reasoning_effort=settings.reasoning_effort) + model = ChatOpenAI(**model_options) world = EvaluationWorld(scenario_id) started = time.monotonic() capture: dict[str, Any] = {} diff --git a/scripts/evaluation/models.py b/scripts/evaluation/models.py index 6e88869e1..c415f4e43 100644 --- a/scripts/evaluation/models.py +++ b/scripts/evaluation/models.py @@ -24,6 +24,7 @@ class ModelSettings: max_output_tokens: int = 8192 timeout_seconds: int = 180 context_window: int = 128000 + wire_api: str = "responses" def __post_init__(self) -> None: """配置文件和 worker 输入共享校验，不能绕过限额或夹带 URL 凭据。""" @@ -39,6 +40,8 @@ class ModelSettings: raise ValueError("所选 provider 没有可用的显式评测凭据") if self.reasoning_effort not in {"none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra"}: raise ValueError("推理预算名称无效") + if self.wire_api not in {"responses", "chat_completions"}: + raise ValueError("模型 provider 协议无效") for value, minimum, maximum in ( (self.max_model_calls, 1, 64), (self.max_output_tokens, 256, 32768), (self.timeout_seconds, 30, 900), (self.context_window, 4096, 2000000), @@ -50,6 +53,7 @@ class ModelSettings: """公开模型与评测预算，省略凭据、完整连接路径和用户配置文件。""" return { "requested_model": self.model, "provider_host": urlsplit(self.base_url).hostname, + "wire_api": self.wire_api, "reasoning_effort": self.reasoning_effort, "max_model_calls": self.max_model_calls, "max_output_tokens": self.max_output_tokens, "timeout_seconds": self.timeout_seconds, "harness_context_window": self.context_window, @@ -63,8 +67,9 @@ def load_codex_model_settings( """只使用选中 provider 显式配置的 bearer/env 凭据，不挪用其他服务的登录令牌。""" config = tomllib.loads(path.read_text(encoding="utf-8")) provider = config.get("model_providers", {}).get(config.get("model_provider"), {}) - if provider.get("wire_api") != "responses": - raise ValueError("当前真实评测入口要求明确配置的 Responses provider") + wire_api = provider.get("wire_api") + if wire_api not in {"responses", "chat_completions"}: + raise ValueError("当前真实评测入口要求明确配置的 Responses 或 Chat Completions provider") endpoint = provider.get("base_url") selected_model = model or config.get("model") if not isinstance(selected_model, str): @@ -73,7 +78,8 @@ def load_codex_model_settings( if not key and isinstance(provider.get("env_key"), str): key = os.environ.get(provider["env_key"]) effort = reasoning_effort or config.get("model_reasoning_effort", "high") - return ModelSettings(selected_model, endpoint, key, effort, max_model_calls, max_output_tokens, timeout_seconds) + return ModelSettings(selected_model, endpoint, key, effort, max_model_calls, max_output_tokens, + timeout_seconds, wire_api=wire_api) class ModelCallLimitError(RuntimeError): diff --git a/tests/test_agent_evaluation_live.py b/tests/test_agent_evaluation_live.py index d1aa4cbad..84da7b860 100644 --- a/tests/test_agent_evaluation_live.py +++ b/tests/test_agent_evaluation_live.py @@ -213,6 +213,24 @@ async def test_successful_model_reply_still_requires_independent_business_eviden assert worker_boundary.model.parameters["max_tokens"] == model_settings.max_output_tokens +@pytest.mark.asyncio +async def test_chat_completions_provider_uses_google_compatible_request_options(worker_boundary, model_settings): + """Chat Completions provider 不得误走 Responses 参数，推理档位仍要显式传递。""" + model_settings = ModelSettings(**{**asdict(model_settings), "wire_api": "chat_completions"}) + + async def run(_world, model, **_kwargs): + """完成一次离线回调，保留模型构造参数供断言。""" + _record_model_reply(model, successful=True) + return {"execution_success": False, "final_text": "执行失败"} + + worker_boundary.runner = run + await live._run_worker("unknown_download", model_settings) + parameters = worker_boundary.model.parameters + assert parameters["use_responses_api"] is False + assert parameters["reasoning_effort"] == model_settings.reasoning_effort + assert "reasoning" not in parameters + + @pytest.mark.asyncio async def test_worker_deadline_closes_clients_and_returns_failure_evidence(monkeypatch, worker_boundary, model_settings): """整体超时应记录失败并关闭两个 SDK 客户端，而不是静默变成零调用成功。""" diff --git a/tests/test_agent_evaluation_models.py b/tests/test_agent_evaluation_models.py index 70d92e165..9f1304678 100644 --- a/tests/test_agent_evaluation_models.py +++ b/tests/test_agent_evaluation_models.py @@ -29,6 +29,18 @@ def test_model_settings_do_not_expose_explicit_credential(tmp_path): assert "private-test-key" not in repr(settings) assert "private-test-key" not in json.dumps(settings.public_metadata()) assert settings.model == "test-model" + assert settings.wire_api == "responses" + + +def test_chat_completions_provider_is_supported(tmp_path): + """Google 等 OpenAI 兼容 provider 可以明确选择 Chat Completions 协议。""" + path = _config(tmp_path).read_text(encoding="utf-8").replace('wire_api = "responses"', + 'wire_api = "chat_completions"') + config = tmp_path / "chat-completions.toml" + config.write_text(path, encoding="utf-8") + settings = load_codex_model_settings(config) + assert settings.wire_api == "chat_completions" + assert settings.public_metadata()["wire_api"] == "chat_completions" def test_only_named_provider_environment_key_is_used(tmp_path, monkeypatch): @@ -109,7 +121,8 @@ def test_empty_or_partial_usage_does_not_break_response_handling(): ("base_url", "https://model.invalid/v1?api_key=private-test-key"), ("base_url", "https://model.invalid/v1#private-test-key"), ("api_key", ""), ("api_key", " "), ("api_key", "private\ntest"), ("api_key", False), - ("reasoning_effort", "unbounded"), ("max_model_calls", 0), ("max_model_calls", 65), + ("reasoning_effort", "unbounded"), ("wire_api", "unknown"), ("max_model_calls", 0), + ("max_model_calls", 65), ("max_model_calls", True), ("max_model_calls", 2.5), ("max_output_tokens", 255), ("max_output_tokens", 32769), ("max_output_tokens", True), ("timeout_seconds", 29), ("timeout_seconds", 901), ("timeout_seconds", False),