nas/MoviePilot
1
0
Fork 0
mirror of https://github.com/jxxghp/MoviePilot.git synced 2026-10-31 20:34:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(evaluation): support chat completions providers

Browse Source
This commit is contained in:
jxxghp
2026-09-11 06:26:26 +08:00
parent b1639b25b6
commit 7ab8cddccb
5 changed files with 54 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
scripts/evaluation/codex.py
View File

@@ -259,6 +259,8 @@ async def _run_codex(scenario_id: str, settings: ModelSettings, executable: str,
def run_codex(scenario_id: str, settings: ModelSettings, *, executable: str = "codex", probe_only: bool = False) -> dict[str, Any]:
"""仅运行已核对的原生版本，探针不会向真实供应商发送请求。"""
if settings.wire_api != "responses":
raise RuntimeError("原生 Codex 对照目前要求 Responses provider；当前配置是 Chat Completions")
binary = shutil.which(executable)
if binary is None:
raise RuntimeError("未找到原生 Codex CLI")

17
scripts/evaluation/live.py
View File

@@ -97,12 +97,17 @@ async def _run_worker(scenario_id: str, settings: ModelSettings) -> dict[str, An
from scripts.evaluation.world import EvaluationWorld
tracker = ModelUsageTracker(settings.max_model_calls)
model = ChatOpenAI(
model=settings.model, api_key=settings.api_key, base_url=settings.base_url,
use_responses_api=True, reasoning={"effort": settings.reasoning_effort},
max_tokens=settings.max_output_tokens, max_retries=0, timeout=min(120, settings.timeout_seconds),
profile={"max_input_tokens": settings.context_window}, callbacks=[tracker],
)
model_options: dict[str, Any] = {
"model": settings.model, "api_key": settings.api_key, "base_url": settings.base_url,
"max_tokens": settings.max_output_tokens, "max_retries": 0,
"timeout": min(120, settings.timeout_seconds),
"profile": {"max_input_tokens": settings.context_window}, "callbacks": [tracker],
}
if settings.wire_api == "responses":
model_options.update(use_responses_api=True, reasoning={"effort": settings.reasoning_effort})
else:
model_options.update(use_responses_api=False, reasoning_effort=settings.reasoning_effort)
model = ChatOpenAI(**model_options)
world = EvaluationWorld(scenario_id)
started = time.monotonic()
capture: dict[str, Any] = {}

12
scripts/evaluation/models.py
View File

@@ -24,6 +24,7 @@ class ModelSettings:
max_output_tokens: int = 8192
timeout_seconds: int = 180
context_window: int = 128000
wire_api: str = "responses"
def __post_init__(self) -> None:
"""配置文件和 worker 输入共享校验，不能绕过限额或夹带 URL 凭据。"""
@@ -39,6 +40,8 @@ class ModelSettings:
raise ValueError("所选 provider 没有可用的显式评测凭据")
if self.reasoning_effort not in {"none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra"}:
raise ValueError("推理预算名称无效")
if self.wire_api not in {"responses", "chat_completions"}:
raise ValueError("模型 provider 协议无效")
for value, minimum, maximum in (
(self.max_model_calls, 1, 64), (self.max_output_tokens, 256, 32768),
(self.timeout_seconds, 30, 900), (self.context_window, 4096, 2000000),
@@ -50,6 +53,7 @@ class ModelSettings:
"""公开模型与评测预算，省略凭据、完整连接路径和用户配置文件。"""
return {
"requested_model": self.model, "provider_host": urlsplit(self.base_url).hostname,
"wire_api": self.wire_api,
"reasoning_effort": self.reasoning_effort, "max_model_calls": self.max_model_calls,
"max_output_tokens": self.max_output_tokens, "timeout_seconds": self.timeout_seconds,
"harness_context_window": self.context_window,
@@ -63,8 +67,9 @@ def load_codex_model_settings(
"""只使用选中 provider 显式配置的 bearer/env 凭据，不挪用其他服务的登录令牌。"""
config = tomllib.loads(path.read_text(encoding="utf-8"))
provider = config.get("model_providers", {}).get(config.get("model_provider"), {})
if provider.get("wire_api") != "responses":
raise ValueError("当前真实评测入口要求明确配置的 Responses provider")
wire_api = provider.get("wire_api")
if wire_api not in {"responses", "chat_completions"}:
raise ValueError("当前真实评测入口要求明确配置的 Responses 或 Chat Completions provider")
endpoint = provider.get("base_url")
selected_model = model or config.get("model")
if not isinstance(selected_model, str):
@@ -73,7 +78,8 @@ def load_codex_model_settings(
if not key and isinstance(provider.get("env_key"), str):
key = os.environ.get(provider["env_key"])
effort = reasoning_effort or config.get("model_reasoning_effort", "high")
return ModelSettings(selected_model, endpoint, key, effort, max_model_calls, max_output_tokens, timeout_seconds)
return ModelSettings(selected_model, endpoint, key, effort, max_model_calls, max_output_tokens,
timeout_seconds, wire_api=wire_api)
class ModelCallLimitError(RuntimeError):

18
tests/test_agent_evaluation_live.py
View File

@@ -213,6 +213,24 @@ async def test_successful_model_reply_still_requires_independent_business_eviden
assert worker_boundary.model.parameters["max_tokens"] == model_settings.max_output_tokens
@pytest.mark.asyncio
async def test_chat_completions_provider_uses_google_compatible_request_options(worker_boundary, model_settings):
"""Chat Completions provider 不得误走 Responses 参数，推理档位仍要显式传递。"""
model_settings = ModelSettings(**{**asdict(model_settings), "wire_api": "chat_completions"})
async def run(_world, model, **_kwargs):
"""完成一次离线回调，保留模型构造参数供断言。"""
_record_model_reply(model, successful=True)
return {"execution_success": False, "final_text": "执行失败"}
worker_boundary.runner = run
await live._run_worker("unknown_download", model_settings)
parameters = worker_boundary.model.parameters
assert parameters["use_responses_api"] is False
assert parameters["reasoning_effort"] == model_settings.reasoning_effort
assert "reasoning" not in parameters
@pytest.mark.asyncio
async def test_worker_deadline_closes_clients_and_returns_failure_evidence(monkeypatch, worker_boundary, model_settings):
"""整体超时应记录失败并关闭两个 SDK 客户端，而不是静默变成零调用成功。"""

15
tests/test_agent_evaluation_models.py
View File

@@ -29,6 +29,18 @@ def test_model_settings_do_not_expose_explicit_credential(tmp_path):
assert "private-test-key" not in repr(settings)
assert "private-test-key" not in json.dumps(settings.public_metadata())
assert settings.model == "test-model"
assert settings.wire_api == "responses"
def test_chat_completions_provider_is_supported(tmp_path):
"""Google 等 OpenAI 兼容 provider 可以明确选择 Chat Completions 协议。"""
path = _config(tmp_path).read_text(encoding="utf-8").replace('wire_api = "responses"',
'wire_api = "chat_completions"')
config = tmp_path / "chat-completions.toml"
config.write_text(path, encoding="utf-8")
settings = load_codex_model_settings(config)
assert settings.wire_api == "chat_completions"
assert settings.public_metadata()["wire_api"] == "chat_completions"
def test_only_named_provider_environment_key_is_used(tmp_path, monkeypatch):
@@ -109,7 +121,8 @@ def test_empty_or_partial_usage_does_not_break_response_handling():
("base_url", "https://model.invalid/v1?api_key=private-test-key"),
("base_url", "https://model.invalid/v1#private-test-key"),
("api_key", ""), ("api_key", " "), ("api_key", "private\ntest"), ("api_key", False),
("reasoning_effort", "unbounded"), ("max_model_calls", 0), ("max_model_calls", 65),
("reasoning_effort", "unbounded"), ("wire_api", "unknown"), ("max_model_calls", 0),
("max_model_calls", 65),
("max_model_calls", True), ("max_model_calls", 2.5),
("max_output_tokens", 255), ("max_output_tokens", 32769), ("max_output_tokens", True),
("timeout_seconds", 29), ("timeout_seconds", 901), ("timeout_seconds", False),