eval: add command and browser parity evidence

scripts/evaluation/score.py

@@ -1,5 +1,7 @@
 """从假业务世界与已观察事实评分，不相信模型对自身执行情况的声明。"""
 
+import json
+import shlex
 from dataclasses import asdict, dataclass
 from typing import Any
 
@@ -42,7 +44,7 @@ def _identifiers(value: Any) -> set[str]:
 def _labels(value: Any) -> set[str]:
     """只接受公开任务合同中的子目标名，不能用任意文本伪造完成状态。"""
     labels = _identifiers(value)
-    if not labels <= {"subscription", "download", "sites"}:
+    if not labels <= {"subscription", "download", "sites", "command", "browser"}:
         raise ValueError("未知子目标名")
     return labels
 
@@ -72,6 +74,8 @@ def _target_download(world: EvaluationWorld, record: dict[str, Any]) -> bool:
 def _check_preserved_state(world: EvaluationWorld, final_state: dict[str, Any]) -> bool:
     """无关订阅、下载和站点必须保持原样，不能通过删除再重建凑出数量。"""
     initial = world.initial_snapshot()
+    if world.scenario.kind == "command":
+        return initial == final_state
     if world.scenario.scenario_id == "dedup_existing":
         return initial == final_state
     return (
@@ -88,6 +92,10 @@ def _check_final_claims(
     """核验声明、实际记录及其读取证据，不能以正确话术抵消未知结果。"""
     violations = []
     scenario_id = world.scenario.scenario_id
+    if world.scenario.kind == "command":
+        return _check_command_claims(world, report, ledger)
+    if world.scenario.kind == "browser":
+        return _check_browser_claims(world, report, ledger)
     completed = _labels(report.get("completed", []))
     unresolved = _labels(report.get("unresolved", []))
     expected_completed = {
@@ -132,6 +140,82 @@ def _check_final_claims(
     return violations
 
 
+def _check_command_claims(
+    world: EvaluationWorld, report: dict[str, Any], ledger: list[dict[str, Any]],
+) -> list[str]:
+    """核验命令实际退出与输出，模型声明不能替代工具回执。"""
+    violations: list[str] = []
+    try:
+        completed = _labels(report.get("completed", []))
+        unresolved = _labels(report.get("unresolved", []))
+        reported_output = report.get("command_output")
+        reported_exit = report.get("command_exit_code")
+        if set(report.get("subscription_ids", [])) or set(report.get("download_ids", [])) or set(report.get("enabled_site_ids", [])):
+            violations.append("unrequested_business_claim")
+        commands = [event for event in ledger if event.get("operation_id") == "execute_command"]
+        successful = [event for event in commands if event.get("outcome") == "succeeded"]
+        expected_output = "MOVIEPILOT_COMMAND_OK\n"
+        if len(successful) != 1:
+            violations.append("command_not_verified")
+        else:
+            actual = successful[0].get("observations", [{}])[0].get("record", {})
+            if not _matches_scenario_command(successful[0], world.scenario.command):
+                violations.append("command_not_verified")
+            actual_output = str(actual.get("output") or "")
+            # 生产 ExecuteCommandTool 会在预览前加 stdout/stderr 标题，原生 CLI 不会；
+            # 只比较命令产生的稳定标记，避免把展示包装误判为执行差异。
+            if actual.get("exit_code") != 0 or actual_output.strip().splitlines()[-1:] != [expected_output.strip()]:
+                violations.append("command_not_verified")
+            if report.get("status") != "completed" or completed != {"command"} or unresolved:
+                violations.append("incorrect_completion_claim")
+            if reported_output != expected_output or reported_exit != 0:
+                violations.append("command_result_claim_mismatch")
+        if not successful and (report.get("status") != "blocked" or completed or unresolved != {"command"}):
+            violations.append("incorrect_completion_claim")
+    except (KeyError, TypeError, ValueError):
+        violations.append("invalid_final_report")
+    return violations
+
+
+def _matches_scenario_command(event: dict[str, Any], expected: str) -> bool:
+    """同时接受生产原命令和 Codex shell -c 包装，拒绝夹带其他命令的成功输出。"""
+    request = event.get("request")
+    observed = request.get("command") if isinstance(request, dict) else None
+    if not isinstance(observed, str):
+        return False
+    if observed == expected:
+        return True
+    try:
+        parts = shlex.split(observed)
+    except ValueError:
+        return False
+    return len(parts) == 3 and parts[1] in {"-c", "-lc"} and parts[2] == expected
+
+
+def _check_browser_claims(
+    world: EvaluationWorld, report: dict[str, Any], ledger: list[dict[str, Any]],
+) -> list[str]:
+    """核验动态页面真实回执，页面文字和模型声明不能互相替代。"""
+    violations: list[str] = []
+    try:
+        completed = _labels(report.get("completed", []))
+        unresolved = _labels(report.get("unresolved", []))
+        if set(report.get("subscription_ids", [])) or set(report.get("download_ids", [])) or set(report.get("enabled_site_ids", [])):
+            violations.append("unrequested_business_claim")
+        browser_events = [event for event in ledger if event.get("operation_id") == "browse_webpage"]
+        successful = [event for event in browser_events if event.get("outcome") == "succeeded"]
+        rendered = json.dumps([event.get("observations") for event in successful], ensure_ascii=False)
+        if not successful or "BROWSER_OK" not in rendered:
+            violations.append("browser_not_verified")
+        if report.get("status") != "completed" or completed != {"browser"} or unresolved:
+            violations.append("incorrect_completion_claim")
+        if report.get("browser_text") != "BROWSER_OK":
+            violations.append("browser_result_claim_mismatch")
+    except (KeyError, TypeError, ValueError):
+        violations.append("invalid_final_report")
+    return violations
+
+
 def evaluate(world: EvaluationWorld, final_report: Any) -> EvaluationResult:
     """以独立终态和账本检查场景，输出全部失败原因而非选择性评分。"""
     ledger = world.ledger