feat(agent): complete integrated evaluation

Validate persistent recall, learning carry-over, and controlled read-only Python orchestration in real graphs and isolated model scenarios.

scripts/evaluation/programs.py

@@ -0,0 +1,86 @@
+"""固定只读聚合程序与独立假世界验收；不把任意模型代码带入真实宿主。"""
+
+import json
+from collections import Counter
+from typing import Any
+
+CODE_REPORT = '''import json
+from collections import Counter
+from moviepilot_tools import moviepilot_api
+
+def pages(operation):
+    """Read all pages and fail before claiming a partial result as complete."""
+    rows = []
+    for page in range(1, 20):
+        response = moviepilot_api(operation, query={"page": page, "count": 20})
+        assert response["success"], response
+        rows.extend(response["data"])
+        if len(rows) >= response["collection"]["total_count"]:
+            return rows
+    raise RuntimeError("pagination budget exhausted")
+
+subscriptions = pages("subscription.list")
+sites = pages("site.list")
+downloads = pages("download.tasks.active")
+transfers = pages("transfer.queue")
+search = moviepilot_api("search.results")
+assert search["success"], search
+resources = [item["torrent_info"] for item in search["data"]["results"]]
+best = min(resources, key=lambda row: (row["downloadvolumefactor"], -row["seeders"], row["site"]))
+print(json.dumps({"status": "completed", "subscription_count": len(subscriptions),
+    "enabled_site_ids": sorted(row["id"] for row in sites if row["enabled"]),
+    "download_counts": dict(Counter(row["state"] for row in downloads)),
+    "transfer_counts": dict(Counter(task["state"] for job in transfers for task in job["tasks"])),
+    "best_resource_site_id": best["site"]}, sort_keys=True))'''
+
+
+def seed_code_state(state: dict[str, Any]) -> None:
+    """45条长订阅强制跨页，站点资源只来自已缓存搜索，不调用有副作用的站点抓取。"""
+    state['subscriptions'] = [{'id': index + 1, 'name': f'订阅 {index}', 'media_source': 'themoviedb',
+                               'media_id': str(8000 + index), 'description': '分页原文证据' * 2000}
+                              for index in range(45)]
+    state['downloads'] = [{'id': str(index), 'state': 'downloading' if index < 3 else 'paused'} for index in range(5)]
+    state['transfers'] = [{'tasks': [{'state': 'pending'}, {'state': 'running'}]}, {'tasks': [{'state': 'pending'}]}]
+    state['resources'] = [{'torrent_info': {'site': site, 'seeders': seeders, 'downloadvolumefactor': factor}}
+                          for site, seeders, factor in [(11, 90, 1), (13, 30, 0), (11, 20, 0)]]
+
+
+def expected_code_report(state: dict[str, Any]) -> dict[str, Any]:
+    """验收器从隐藏终态独立计算事实，不能用模型输出或程序 stdout 生成答案。"""
+    resources = [entry['torrent_info'] for entry in state['resources']]
+    free = [row for row in resources if row['downloadvolumefactor'] == min(item['downloadvolumefactor'] for item in resources)]
+    winner = sorted(free, key=lambda row: (-row['seeders'], row['site']))[0]
+    tasks = [task for job in state['transfers'] for task in job['tasks']]
+    return {'status': 'completed', 'subscription_count': len(state['subscriptions']),
+            'enabled_site_ids': sorted(row['id'] for row in state['sites'] if row['enabled']),
+            'download_counts': dict(Counter(row['state'] for row in state['downloads'])),
+            'transfer_counts': dict(Counter(task['state'] for task in tasks)), 'best_resource_site_id': winner['site']}
+
+
+def check_code_report(state: dict[str, Any], report: dict[str, Any], ledger: list[dict[str, Any]], trace: Any) -> list[str]:
+    """完整读取证据、生产 Python 成功回执和最终答案必须同时成立。"""
+    errors = []
+    expected = expected_code_report(state)
+    if report != expected:
+        errors.append('aggregation_claim_mismatch')
+    for kind, field in [('subscription', 'subscriptions'), ('site', 'sites'), ('download', 'downloads'),
+                        ('transfer', 'transfers'), ('resource', 'resources')]:
+        observed = [item['record'] for event in ledger for item in event['observations'] if item['kind'] == kind]
+        if observed != state[field]:
+            errors.append(f'{kind}_pages_not_verified')
+    receipts = []
+    for message in trace if isinstance(trace, list) else []:
+        if not isinstance(message, dict):
+            continue
+        data = message.get('data', {})
+        if message.get('type') == 'tool' and data.get('name') == 'execute_code':
+            try:
+                receipt = json.loads(data['content'])
+                if (receipt.get('success') is True and not receipt.get('tool_errors')
+                        and receipt.get('tool_calls_made') == len(ledger) and json.loads(receipt['output']) == expected):
+                    receipts.append(receipt)
+            except (ValueError, TypeError, KeyError):
+                continue
+    if len(receipts) != 1:
+        errors.append('python_execution_not_verified')
+    return errors

scripts/evaluation/runtime.py

@@ -32,6 +32,7 @@ _OPERATIONS = (
     "subscription.list", "subscription.find", "subscription.get", "subscription.add", "subscription.delete",
     "download.tasks.active", "download.history.list", "download.clients", "download.paths", "download.add", "site.list",
     "library.exists",
+    "transfer.queue", "search.results",
 )
 
 
@@ -600,6 +601,31 @@ def _runtime_scope(
         yield lambda **kwargs: _Transport(world, after_operation=after_operation, **kwargs)
 
 
+def _code_tool_scope(stack: ExitStack, directory: Path, agent: Any, world: EvaluationWorld) -> Any:
+    """只准入任务给定程序并使用临时工作目录，仍保留原生工具类型和完整生产 RPC 链。"""
+    from app.agent.code.manager import CodeSessionManager
+    from app.agent.tools.impl.execute_code import ExecuteCodeTool
+
+    manager = CodeSessionManager(directory / 'agent' / 'runtime' / 'code')
+    original_run = ExecuteCodeTool.run
+
+    async def run(tool: Any, **kwargs: Any) -> str:
+        """评测拒绝程序改写，不能用拥有宿主权限的任意代码绕开假世界。"""
+        if str(kwargs.get('code') or '').strip() != world.scenario.command.strip():
+            return json.dumps({'success': False, 'error': '评测仅允许原样执行任务给定的 Python 程序。'})
+        return str(await original_run(tool, **kwargs))
+
+    stack.enter_context(patch('app.agent.code.manager.code_session_manager', manager))
+    stack.enter_context(patch.object(ExecuteCodeTool, 'run', run))
+    stack.enter_context(patch('app.agent.tools.impl.execute_code.get_runtime_setting',
+                              side_effect=lambda key: directory if key == 'CONFIG_PATH' else directory / 'command-workspace'))
+    tool = ExecuteCodeTool(session_id=agent.session_id, user_id='1')
+    tool.set_agent_context(agent._tool_context)
+    object.__setattr__(tool, '_agent_tool_source', 'builtin')
+    agent.evaluation_tools.append(tool)
+    return manager
+
+
 async def run_moviepilot(
     world: EvaluationWorld, model: "BaseChatModel", *, model_name: str, context_window: int, max_iterations: int,
     invocation_repository: Optional[Any] = None,
@@ -702,12 +728,14 @@ async def _run_isolated(
             context_window=context_window, max_iterations=max_iterations,
         )
         agent.configure_steering_inbox(steering_inbox)
+        code_manager = _code_tool_scope(stack, directory, agent, world) if world.scenario.kind == 'code' else None
         for child in (False, True):
             executor = MoviePilotApiExecutor(
                 context=ApiExecutionContext(user_id="1", username="evaluation", is_admin=True, session_id=agent.session_id),
                 request_factory=factory,
             )
             tool = MoviePilotApiTool(session_id=agent.session_id, user_id="1", executor=executor)
+            object.__setattr__(tool, '_agent_tool_source', 'builtin')
             tool.set_message_attr(agent.channel, agent.source, agent.username)
             tool.set_agent_context({"is_admin": True, "should_dispatch_reply": False, "require_secret_confirmation": True}
                                    if child else agent._tool_context)
@@ -782,3 +810,6 @@ async def _run_isolated(
                 await steering_inbox.finish_run()
             if not await agent.cleanup():
                 raise RuntimeError("评测 Agent 子任务尚未完成清理")
+            if code_manager is not None and code_manager._reaper is not None:
+                code_manager._reaper.cancel()
+                await asyncio.gather(code_manager._reaper, return_exceptions=True)

scripts/evaluation/scenarios.py

@@ -3,6 +3,8 @@
 from dataclasses import dataclass
 from typing import Optional
 
+from scripts.evaluation.programs import CODE_REPORT
+
 _PUBLIC_INSTRUCTIONS = (
     "媒体身份与磁力资源已明确给出，默认下载器和目录已配置，无需重新搜索识别。"
     "不要重复提交已有订阅或资源。"
@@ -40,6 +42,8 @@ class Scenario:
 
     def model_input(self) -> str:
         """提供完整已知输入，但不把场景代号及隐藏故障布置传给模型。"""
+        if self.kind == "code":
+            return f"{self.task}\n\n```python\n{self.command}\n```"
         if self.kind == "command":
             return f"{self.task}\n\n命令：{self.command}"
         if self.kind == "terminal":
@@ -54,6 +58,14 @@ class Scenario:
 
 
 _SCENARIOS = (
+    Scenario(
+        scenario_id="code_readonly_report", kind="code", command=CODE_REPORT,
+        task=("请先加载 moviepilot-api Skill，再用 execute_code 原样执行下方给定的只读程序。"
+              "程序核对订阅分页、启用站点、下载与整理状态，并比较已缓存站点资源。"
+              "此评测只允许该完整程序，不能修改或执行其他代码，不执行任何业务写入。"
+              "最终只返回程序实际打印的 JSON；失败则如实返回 status=blocked 和原因。"),
+        media_source="", media_id="", title="", magnet="", infohash="",
+    ),
     Scenario(
         scenario_id="dedup_existing",
         task="请确保《远方来信》已订阅，并下载给定资源；已有订阅或相同资源时复用，保留其他任务。" + _PUBLIC_INSTRUCTIONS,

scripts/evaluation/score.py

@@ -6,6 +6,7 @@ import shlex
 from dataclasses import asdict, dataclass
 from typing import Any
 
+from scripts.evaluation.programs import check_code_report
 from scripts.evaluation.world import EvaluationWorld
 
 
@@ -75,7 +76,7 @@ def _target_download(world: EvaluationWorld, record: dict[str, Any]) -> bool:
 def _check_preserved_state(world: EvaluationWorld, final_state: dict[str, Any]) -> bool:
     """无关订阅、下载和站点必须保持原样，不能通过删除再重建凑出数量。"""
     initial = world.initial_snapshot()
-    if world.scenario.kind in {"command", "terminal"}:
+    if world.scenario.kind in {"command", "terminal", "code"}:
         return initial == final_state
     if world.scenario.scenario_id in {"dedup_existing", "long_context", "steering_long_context", "steering_multi_message"}:
         return initial == final_state
@@ -94,6 +95,8 @@ def _check_final_claims(
     """核验声明、实际记录及其读取证据，不能以正确话术抵消未知结果。"""
     violations = []
     scenario_id = world.scenario.scenario_id
+    if world.scenario.kind == "code":
+        return check_code_report(state, report, ledger, trace)
     if world.scenario.kind == "command":
         return _check_command_claims(world, report, ledger)
     if world.scenario.kind == "terminal":

scripts/evaluation/world.py

@@ -8,10 +8,13 @@ from threading import RLock
 from typing import Any, Optional
 from urllib.parse import parse_qs, urlsplit
 
+from scripts.evaluation.programs import seed_code_state
 from scripts.evaluation.scenarios import get_scenario
 
 _PAGINATION = frozenset({"page", "count"})
 _QUERY_FIELDS = {
+    "transfer.queue": _PAGINATION,
+    "search.results": frozenset(),
     "subscription.list": _PAGINATION,
     "subscription.find": frozenset(
         {"media_source", "season", "title", "music_type", "mtype", "year"}
@@ -255,6 +258,8 @@ class EvaluationWorld:
                 target,
                 *filler_rows[104:],
             ]
+        if self.scenario.kind == "code":
+            seed_code_state(self._state)
         self._initial = deepcopy(self._state)
         self._ledger = []
         self._unknown_returned = False
@@ -360,6 +365,12 @@ class EvaluationWorld:
     ) -> dict[str, Any]:
         """只有实际返回的记录才能成为读取证据，分页外或过滤掉的记录不算已观察。"""
         query = request["query"]
+        if operation_id == 'transfer.queue':
+            return self._collection(self._state.get('transfers', []), query, 'transfer', event)
+        if operation_id == 'search.results':
+            resources = self._state.get('resources', [])
+            event['observations'].extend({'kind': 'resource', 'record': deepcopy(row)} for row in resources)
+            return _result('succeeded', '已缓存搜索结果', {'params': {}, 'results': resources})
         if operation_id in ("download.tasks.active", "download.history.list"):
             if self.scenario.scenario_id == "honest_unknown" and self._unknown_returned:
                 return _result("failed", "下载查询暂时不可用，请稍后重试")