diff --git a/app/agent/prompt/System Core Prompt.txt b/app/agent/prompt/System Core Prompt.txt index 79b9e75ca..3e7c62ff9 100644 --- a/app/agent/prompt/System Core Prompt.txt +++ b/app/agent/prompt/System Core Prompt.txt @@ -35,7 +35,7 @@ You act as a proactive agent. Your goal is to fully resolve the user's media-rel - Treat manual download and subscription automation as two execution modes of the same acquisition pipeline. Manual download is user-triggered immediate acquisition; subscription is persistent site-driven monitoring and acquisition. - Keep the user anchored to the operational step that matters now: site, search, recognition, download, subscription, transfer, or status/history. - Users may attach images from supported channels; analyze them together with the text when relevant. -- When visual inspection is needed for an image URL or local image path, call `view_image` with exactly one of `url` or `file_path`; only describe visual details after the tool returns an image observation. +- When visual inspection is needed for an image URL, local image path, or supplied image content, call `view_image` with exactly one of `url`, `file_path`, or `image_data`; only describe visual details after the tool returns an image observation. - User messages may arrive as structured JSON. Treat the `message` field as the user's text. Input metadata appears in `input`; when `input.mode` is `voice`, the user sent a voice message and `message` contains its transcript. Attachments appear in `files`; when `local_path` is present, use local file tools to inspect the uploaded file directly. When image input is disabled for the current model, user images may also be delivered through `files`. diff --git a/app/agent/tools/impl/view_image.py b/app/agent/tools/impl/view_image.py index dd9d6a902..2f68d2fac 100644 --- a/app/agent/tools/impl/view_image.py +++ b/app/agent/tools/impl/view_image.py @@ -45,7 +45,7 @@ class ViewImageError(ValueError): self.message = message -class ViewImageInput(BaseModel): +class ViewImageInput(BaseModel): # type: ignore[misc] """图片查看工具的输入参数模型。""" url: Optional[str] = Field( @@ -58,8 +58,13 @@ class ViewImageInput(BaseModel): file_path: Optional[str] = Field( None, description=( - "Local image path. Non-administrator users may only read files under " - "the Agent configuration directory." + "Local image path. Non-administrator users may only read files under the Agent configuration directory." + ), + ) + image_data: Optional[Union[str, bytes]] = Field( + None, + description=( + "Base64 image content, a data:image/...;base64,... URL, or raw image bytes. Use exactly one image source." ), ) detail: Literal["auto", "low", "high"] = Field( @@ -69,11 +74,12 @@ class ViewImageInput(BaseModel): @model_validator(mode="after") # type: ignore[misc] def require_one_source(self) -> "ViewImageInput": - """确保图片来源恰好是 URL 或本地路径中的一个。""" + """确保图片来源恰好是 URL、本地路径或图片内容中的一个。""" has_url = bool(self.url and self.url.strip()) has_file_path = bool(self.file_path and self.file_path.strip()) - if has_url == has_file_path: - raise ValueError("url and file_path are mutually exclusive; provide exactly one") + has_image_data = bool(self.image_data) + if sum((has_url, has_file_path, has_image_data)) != 1: + raise ValueError("provide exactly one of url, file_path, or image_data") return self @@ -88,8 +94,8 @@ class ViewImageTool(MoviePilotTool): ] description: str = ( "View an image so a multimodal model can inspect its actual pixels. " - "Provide exactly one of url or file_path. Supports public HTTP(S) image URLs, " - "data:image/...;base64,... URLs, and local image paths. Local access follows " + "Provide exactly one of url, file_path, or image_data. Supports public HTTP(S) " + "image URLs, data:image/...;base64,... URLs, Base64/raw image content, and local image paths. Local access follows " "the Agent file permission boundary; private-network URLs are blocked. " "Use this when visual inspection is needed, and do not claim to have seen " "the image unless the tool returns an image observation." @@ -100,6 +106,7 @@ class ViewImageTool(MoviePilotTool): """根据图片来源生成不泄露查询参数的工具提示。""" file_path = str(kwargs.get("file_path") or "").strip() url = str(kwargs.get("url") or "").strip() + image_data = kwargs.get("image_data") if file_path: return f"查看本地图片: {Path(file_path).name or '未知文件'}" if url.lower().startswith("data:image/"): @@ -111,6 +118,9 @@ class ViewImageTool(MoviePilotTool): except ValueError: label = "远程图片 URL" return f"查看远程图片: {label[:256]}" + if image_data: + size = len(image_data) if hasattr(image_data, "__len__") else 0 + return f"查看图片内容: {size} chars/bytes" return "查看图片" def format_agent_result(self, result: Any, **tool_arguments: Any) -> Union[str, list[dict[str, Any]]]: @@ -171,6 +181,7 @@ class ViewImageTool(MoviePilotTool): self, url: Optional[str] = None, file_path: Optional[str] = None, + image_data: Optional[Union[str, bytes]] = None, detail: Literal["auto", "low", "high"] = "auto", **kwargs: Any, ) -> str: @@ -178,15 +189,17 @@ class ViewImageTool(MoviePilotTool): del kwargs clean_url = str(url or "").strip() clean_file_path = str(file_path or "").strip() + clean_image_data = image_data has_url = bool(clean_url) has_file_path = bool(clean_file_path) - if has_url == has_file_path: + has_image_data = bool(clean_image_data) + if sum((has_url, has_file_path, has_image_data)) != 1: return self._failure( "invalid_source", - "请在 url 和 file_path 中恰好提供一个图片来源。", + "请在 url、file_path 和 image_data 中恰好提供一个图片来源。", ) - source_type = "url" if has_url else "file" + source_type = "url" if has_url else "file" if has_file_path else "content" try: if has_file_path: resolved_path, access_error = await self._check_local_file_access( @@ -202,6 +215,11 @@ class ViewImageTool(MoviePilotTool): elif clean_url.lower().startswith("data:image/"): image_bytes = self._decode_data_url(clean_url) source_label = "data image" + elif has_image_data: + if clean_image_data is None: + raise ViewImageError("invalid_image", "图片内容不能为空。") + image_bytes = self._decode_image_content(clean_image_data) + source_label = "image content" else: await self._validate_remote_url(clean_url) image_bytes = await self._fetch_remote_image(clean_url) @@ -224,7 +242,7 @@ class ViewImageTool(MoviePilotTool): ) except ViewImageError as error: return self._failure(error.code, error.message, source_type) - except (FileNotFoundError, IsADirectoryError, PermissionError): + except FileNotFoundError, IsADirectoryError, PermissionError: return self._failure("local_file_unavailable", "本地图片不存在、不是文件或没有读取权限。", source_type) except Exception as error: # noqa: BLE001 - 图片读取边界必须稳定返回失败合同 logger.warning("Agent 查看图片失败: %s", type(error).__name__) @@ -328,6 +346,28 @@ class ViewImageTool(MoviePilotTool): raise ViewImageError("invalid_image", "图片 data URL 为空。") return image_bytes + @staticmethod + def _decode_image_content(content: Union[str, bytes]) -> bytes: + """解码纯 Base64、data URL 或原始图片字节，并限制输入大小。""" + if isinstance(content, bytes): + image_bytes = content + elif isinstance(content, str): + value = content.strip() + if value.lower().startswith("data:"): + image_bytes = ViewImageTool._decode_data_url(value) + else: + if len(value) > IMAGE_MAX_BASE64_CHARS: + raise ViewImageError("image_too_large", "图片编码超过大小上限。") + try: + image_bytes = base64.b64decode("".join(value.split()), validate=True) + except (ValueError, binascii.Error) as error: + raise ViewImageError("invalid_image", "图片 Base64 内容无效。") from error + else: + raise ViewImageError("invalid_image", "图片内容必须是 Base64 文本或原始字节。") + if not image_bytes or len(image_bytes) > IMAGE_MAX_BYTES: + raise ViewImageError("image_too_large", "图片为空或超过大小上限。") + return image_bytes + @staticmethod def _validate_image_bytes(image_bytes: bytes) -> tuple[str, int, int]: """验证真实图片格式、像素和解码结果，防止伪造 MIME 或损坏数据进入模型。""" diff --git a/docs/agent.md b/docs/agent.md index 90fea276d..6f65980e0 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -54,7 +54,7 @@ WebAgent 在 Agent 正在运行时仍可提交文本和附件。宿主为每个 ## 工具图片与模型视觉 -`browse_webpage(action="screenshot")` 和 `view_image(url=...|file_path=...)` 在 Agent 中返回真实图像块和有限来源说明。图像在通用文本截断之前处理，最终请求按完整工具回复批次附加带工具来源的临时图像观察，兼容 Chat、Responses、Anthropic 与 Gemini 的图片输入；原始用户消息、工具调用 ID 和授权不变。`view_image` 的远程 URL 只允许通过公网安全校验的 HTTP(S) 地址，本地路径继续遵守 Agent 文件访问边界。 +`browse_webpage(action="screenshot")` 和 `view_image(url=...|file_path=...|image_data=...)` 在 Agent 中返回真实图像块和有限来源说明。图像在通用文本截断之前处理，最终请求按完整工具回复批次附加带工具来源的临时图像观察，兼容 Chat、Responses、Anthropic 与 Gemini 的图片输入；原始用户消息、工具调用 ID 和授权不变。`view_image` 的远程 URL 只允许通过公网安全校验的 HTTP(S) 地址，本地路径继续遵守 Agent 文件访问边界，`image_data` 支持纯 Base64、data URL 和原始字节。 每次模型请求都检查实际模型资料和 `LLM_SUPPORT_IMAGE_INPUT` 开关。已知纯文本模型会得到明确的“未接收工具图片”说明。服务明确拒绝图片时，只对该次模型调用做一次文字回退，随后本轮沿用文字观察，不重复执行图片工具或其他工具；认证、限流等错误保持原来的错误语义。临时观察不会写入会话历史。 diff --git a/docs/mcp-api.md b/docs/mcp-api.md index 3965cf5f0..29ac41db7 100644 --- a/docs/mcp-api.md +++ b/docs/mcp-api.md @@ -552,7 +552,7 @@ MCP、HTTP 工具管理接口、本地 CLI 和内置 Agent 都从同一严格目 内置 Agent 在专用格式化路径把成功截图转换为图像输入，外部 HTTP/MCP 客户端仍按 原 JSON 合同消费；本次不宣称外部 MCP 已提供原生 image content block。 -`view_image` 只供内置 Agent 使用：它会在 Agent 专用格式化路径把 URL 或本地图片转换为 +`view_image` 只供内置 Agent 使用：它会在 Agent 专用格式化路径把 URL、本地图片或图片内容转换为 原生图像输入，HTTP/MCP 直调不提供该工具，避免把只能由视觉中间件消费的图像块误当作普通 JSON。 内置命令工具 `execute_command(action="run")` 的返回值为 JSON 字符串，包含 diff --git a/tests/test_agent_view_image.py b/tests/test_agent_view_image.py index 30f4e2e2f..194827626 100644 --- a/tests/test_agent_view_image.py +++ b/tests/test_agent_view_image.py @@ -73,6 +73,7 @@ def test_input_requires_exactly_one_image_source() -> None: with pytest.raises(ValidationError): ViewImageInput(url="https://images.example.invalid/a.png", file_path="/tmp/a.png") assert ViewImageInput(url="https://images.example.invalid/a.png").detail == "auto" + assert ViewImageInput(image_data=base64.b64encode(_image_bytes()).decode("ascii")).detail == "auto" @pytest.mark.asyncio @@ -112,6 +113,23 @@ async def test_data_url_is_validated_and_projected_without_network() -> None: assert projected[1]["image_url"]["url"] == f"data:image/jpeg;base64,{encoded}" +@pytest.mark.asyncio +async def test_image_content_accepts_raw_bytes_and_plain_base64() -> None: + """图片内容输入支持工具间传递的原始字节和纯 Base64 文本。""" + image = _image_bytes("PNG") + tool = _tool() + + raw_result = await tool.run(image_data=image) + encoded_result = await tool.run(image_data=base64.b64encode(image).decode("ascii")) + + raw_payload = json.loads(raw_result) + encoded_payload = json.loads(encoded_result) + assert raw_payload["source_type"] == "content" + assert encoded_payload["source_type"] == "content" + assert base64.b64decode(raw_payload["image_base64"]) == image + assert base64.b64decode(encoded_payload["image_base64"]) == image + + @pytest.mark.asyncio async def test_remote_image_uses_ssrf_validation_and_streaming_download(monkeypatch: pytest.MonkeyPatch) -> None: """远程图片必须先通过公网校验，再使用有大小上限的异步流下载。""" @@ -144,6 +162,7 @@ async def test_remote_image_uses_ssrf_validation_and_streaming_download(monkeypa @pytest.mark.asyncio async def test_remote_private_or_rejected_url_never_reaches_http(monkeypatch: pytest.MonkeyPatch) -> None: """未通过 URL 安全校验时必须在网络边界前失败。""" + async def unsafe_url(*_args: Any, **_kwargs: Any) -> bool: """拒绝模拟的内部目标。""" return False @@ -165,6 +184,7 @@ async def test_remote_private_or_rejected_url_never_reaches_http(monkeypatch: py @pytest.mark.asyncio async def test_remote_content_length_limit_is_enforced_before_body_read(monkeypatch: pytest.MonkeyPatch) -> None: """远程响应声明超限时应立即失败，不把大响应交给图片解码器。""" + async def safe_url(*_args: Any, **_kwargs: Any) -> bool: """放行测试 URL。""" return True @@ -185,9 +205,6 @@ async def test_remote_content_length_limit_is_enforced_before_body_read(monkeypa def test_image_tool_is_registered_for_agent_but_keeps_raw_data_out_of_generic_formatter() -> None: """工具工厂应注册图片能力，成功图片由专用 formatter 保持图块而非文本截断。""" - tool_names = { - tool_class.model_fields["name"].default - for tool_class in MoviePilotToolFactory.BUILTIN_TOOL_CLASSES - } + tool_names = {tool_class.model_fields["name"].default for tool_class in MoviePilotToolFactory.BUILTIN_TOOL_CLASSES} assert "view_image" in tool_names assert IMAGE_MAX_BYTES == 768 * 1024