nas/MoviePilot
1
0
Fork 0
mirror of https://github.com/jxxghp/MoviePilot.git synced 2026-10-31 20:34:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(agent): accept image content in view_image

Browse Source
This commit is contained in:
jxxghp
2026-09-11 23:33:04 +08:00
parent 80b510a6a5
commit 9cc2ab08ad
5 changed files with 76 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/agent/prompt/System Core Prompt.txt
View File

@@ -35,7 +35,7 @@ You act as a proactive agent. Your goal is to fully resolve the user's media-rel
- Treat manual download and subscription automation as two execution modes of the same acquisition pipeline. Manual download is user-triggered immediate acquisition; subscription is persistent site-driven monitoring and acquisition.
- Keep the user anchored to the operational step that matters now: site, search, recognition, download, subscription, transfer, or status/history.
- Users may attach images from supported channels; analyze them together with the text when relevant.
- When visual inspection is needed for an image URL or local image path, call `view_image` with exactly one of `url` or `file_path`; only describe visual details after the tool returns an image observation.
- When visual inspection is needed for an image URL, local image path, or supplied image content, call `view_image` with exactly one of `url`, `file_path`, or `image_data`; only describe visual details after the tool returns an image observation.
- User messages may arrive as structured JSON. Treat the `message` field as the user's text. Input metadata appears in `input`; when `input.mode` is `voice`, the user sent a voice message and `message` contains its transcript. Attachments appear in `files`; when `local_path` is present, use local file tools to inspect the uploaded file directly. When image input is disabled for the current model, user images may also be delivered through `files`.
</moviepilot_domain_model>

64
app/agent/tools/impl/view_image.py
View File

@@ -45,7 +45,7 @@ class ViewImageError(ValueError):
self.message = message
class ViewImageInput(BaseModel):
class ViewImageInput(BaseModel): # type: ignore[misc]
"""图片查看工具的输入参数模型。"""
url: Optional[str] = Field(
@@ -58,8 +58,13 @@ class ViewImageInput(BaseModel):
file_path: Optional[str] = Field(
None,
description=(
"Local image path. Non-administrator users may only read files under "
"the Agent configuration directory."
"Local image path. Non-administrator users may only read files under the Agent configuration directory."
),
)
image_data: Optional[Union[str, bytes]] = Field(
None,
description=(
"Base64 image content, a data:image/...;base64,... URL, or raw image bytes. Use exactly one image source."
),
)
detail: Literal["auto", "low", "high"] = Field(
@@ -69,11 +74,12 @@ class ViewImageInput(BaseModel):
@model_validator(mode="after") # type: ignore[misc]
def require_one_source(self) -> "ViewImageInput":
"""确保图片来源恰好是 URL 或本地路径中的一个。"""
"""确保图片来源恰好是 URL、本地路径或图片内容中的一个。"""
has_url = bool(self.url and self.url.strip())
has_file_path = bool(self.file_path and self.file_path.strip())
if has_url == has_file_path:
raise ValueError("url and file_path are mutually exclusive; provide exactly one")
has_image_data = bool(self.image_data)
if sum((has_url, has_file_path, has_image_data)) != 1:
raise ValueError("provide exactly one of url, file_path, or image_data")
return self
@@ -88,8 +94,8 @@ class ViewImageTool(MoviePilotTool):
]
description: str = (
"View an image so a multimodal model can inspect its actual pixels. "
"Provide exactly one of url or file_path. Supports public HTTP(S) image URLs, "
"data:image/...;base64,... URLs, and local image paths. Local access follows "
"Provide exactly one of url, file_path, or image_data. Supports public HTTP(S) "
"image URLs, data:image/...;base64,... URLs, Base64/raw image content, and local image paths. Local access follows "
"the Agent file permission boundary; private-network URLs are blocked. "
"Use this when visual inspection is needed, and do not claim to have seen "
"the image unless the tool returns an image observation."
@@ -100,6 +106,7 @@ class ViewImageTool(MoviePilotTool):
"""根据图片来源生成不泄露查询参数的工具提示。"""
file_path = str(kwargs.get("file_path") or "").strip()
url = str(kwargs.get("url") or "").strip()
image_data = kwargs.get("image_data")
if file_path:
return f"查看本地图片: {Path(file_path).name or '未知文件'}"
if url.lower().startswith("data:image/"):
@@ -111,6 +118,9 @@ class ViewImageTool(MoviePilotTool):
except ValueError:
label = "远程图片 URL"
return f"查看远程图片: {label[:256]}"
if image_data:
size = len(image_data) if hasattr(image_data, "__len__") else 0
return f"查看图片内容: {size} chars/bytes"
return "查看图片"
def format_agent_result(self, result: Any, **tool_arguments: Any) -> Union[str, list[dict[str, Any]]]:
@@ -171,6 +181,7 @@ class ViewImageTool(MoviePilotTool):
self,
url: Optional[str] = None,
file_path: Optional[str] = None,
image_data: Optional[Union[str, bytes]] = None,
detail: Literal["auto", "low", "high"] = "auto",
**kwargs: Any,
) -> str:
@@ -178,15 +189,17 @@ class ViewImageTool(MoviePilotTool):
del kwargs
clean_url = str(url or "").strip()
clean_file_path = str(file_path or "").strip()
clean_image_data = image_data
has_url = bool(clean_url)
has_file_path = bool(clean_file_path)
if has_url == has_file_path:
has_image_data = bool(clean_image_data)
if sum((has_url, has_file_path, has_image_data)) != 1:
return self._failure(
"invalid_source",
"请在 url 和 file_path 中恰好提供一个图片来源。",
"请在 url、file_path 和 image_data 中恰好提供一个图片来源。",
)
source_type = "url" if has_url else "file"
source_type = "url" if has_url else "file" if has_file_path else "content"
try:
if has_file_path:
resolved_path, access_error = await self._check_local_file_access(
@@ -202,6 +215,11 @@ class ViewImageTool(MoviePilotTool):
elif clean_url.lower().startswith("data:image/"):
image_bytes = self._decode_data_url(clean_url)
source_label = "data image"
elif has_image_data:
if clean_image_data is None:
raise ViewImageError("invalid_image", "图片内容不能为空。")
image_bytes = self._decode_image_content(clean_image_data)
source_label = "image content"
else:
await self._validate_remote_url(clean_url)
image_bytes = await self._fetch_remote_image(clean_url)
@@ -224,7 +242,7 @@ class ViewImageTool(MoviePilotTool):
)
except ViewImageError as error:
return self._failure(error.code, error.message, source_type)
except (FileNotFoundError, IsADirectoryError, PermissionError):
except FileNotFoundError, IsADirectoryError, PermissionError:
return self._failure("local_file_unavailable", "本地图片不存在、不是文件或没有读取权限。", source_type)
except Exception as error: # noqa: BLE001 - 图片读取边界必须稳定返回失败合同
logger.warning("Agent 查看图片失败: %s", type(error).__name__)
@@ -328,6 +346,28 @@ class ViewImageTool(MoviePilotTool):
raise ViewImageError("invalid_image", "图片 data URL 为空。")
return image_bytes
@staticmethod
def _decode_image_content(content: Union[str, bytes]) -> bytes:
"""解码纯 Base64、data URL 或原始图片字节，并限制输入大小。"""
if isinstance(content, bytes):
image_bytes = content
elif isinstance(content, str):
value = content.strip()
if value.lower().startswith("data:"):
image_bytes = ViewImageTool._decode_data_url(value)
else:
if len(value) > IMAGE_MAX_BASE64_CHARS:
raise ViewImageError("image_too_large", "图片编码超过大小上限。")
try:
image_bytes = base64.b64decode("".join(value.split()), validate=True)
except (ValueError, binascii.Error) as error:
raise ViewImageError("invalid_image", "图片 Base64 内容无效。") from error
else:
raise ViewImageError("invalid_image", "图片内容必须是 Base64 文本或原始字节。")
if not image_bytes or len(image_bytes) > IMAGE_MAX_BYTES:
raise ViewImageError("image_too_large", "图片为空或超过大小上限。")
return image_bytes
@staticmethod
def _validate_image_bytes(image_bytes: bytes) -> tuple[str, int, int]:
"""验证真实图片格式、像素和解码结果，防止伪造 MIME 或损坏数据进入模型。"""

2
docs/agent.md
View File

@@ -54,7 +54,7 @@ WebAgent 在 Agent 正在运行时仍可提交文本和附件。宿主为每个
## 工具图片与模型视觉
`browse_webpage(action="screenshot")` 和 `view_image(url=...|file_path=...)` 在 Agent 中返回真实图像块和有限来源说明。图像在通用文本截断之前处理，最终请求按完整工具回复批次附加带工具来源的临时图像观察，兼容 Chat、Responses、Anthropic 与 Gemini 的图片输入；原始用户消息、工具调用 ID 和授权不变。`view_image` 的远程 URL 只允许通过公网安全校验的 HTTP(S) 地址，本地路径继续遵守 Agent 文件访问边界。
`browse_webpage(action="screenshot")` 和 `view_image(url=...|file_path=...|image_data=...)` 在 Agent 中返回真实图像块和有限来源说明。图像在通用文本截断之前处理，最终请求按完整工具回复批次附加带工具来源的临时图像观察，兼容 Chat、Responses、Anthropic 与 Gemini 的图片输入；原始用户消息、工具调用 ID 和授权不变。`view_image` 的远程 URL 只允许通过公网安全校验的 HTTP(S) 地址，本地路径继续遵守 Agent 文件访问边界，`image_data` 支持纯 Base64、data URL 和原始字节。
每次模型请求都检查实际模型资料和 `LLM_SUPPORT_IMAGE_INPUT` 开关。已知纯文本模型会得到明确的“未接收工具图片”说明。服务明确拒绝图片时，只对该次模型调用做一次文字回退，随后本轮沿用文字观察，不重复执行图片工具或其他工具；认证、限流等错误保持原来的错误语义。临时观察不会写入会话历史。

2
docs/mcp-api.md
View File

@@ -552,7 +552,7 @@ MCP、HTTP 工具管理接口、本地 CLI 和内置 Agent 都从同一严格目
内置 Agent 在专用格式化路径把成功截图转换为图像输入，外部 HTTP/MCP 客户端仍按
原 JSON 合同消费；本次不宣称外部 MCP 已提供原生 image content block。
`view_image` 只供内置 Agent 使用：它会在 Agent 专用格式化路径把 URL 或本地图片转换为
`view_image` 只供内置 Agent 使用：它会在 Agent 专用格式化路径把 URL、本地图片或图片内容转换为
原生图像输入，HTTP/MCP 直调不提供该工具，避免把只能由视觉中间件消费的图像块误当作普通 JSON。
内置命令工具 `execute_command(action="run")` 的返回值为 JSON 字符串，包含

25
tests/test_agent_view_image.py
View File

@@ -73,6 +73,7 @@ def test_input_requires_exactly_one_image_source() -> None:
with pytest.raises(ValidationError):
ViewImageInput(url="https://images.example.invalid/a.png", file_path="/tmp/a.png")
assert ViewImageInput(url="https://images.example.invalid/a.png").detail == "auto"
assert ViewImageInput(image_data=base64.b64encode(_image_bytes()).decode("ascii")).detail == "auto"
@pytest.mark.asyncio
@@ -112,6 +113,23 @@ async def test_data_url_is_validated_and_projected_without_network() -> None:
assert projected[1]["image_url"]["url"] == f"data:image/jpeg;base64,{encoded}"
@pytest.mark.asyncio
async def test_image_content_accepts_raw_bytes_and_plain_base64() -> None:
"""图片内容输入支持工具间传递的原始字节和纯 Base64 文本。"""
image = _image_bytes("PNG")
tool = _tool()
raw_result = await tool.run(image_data=image)
encoded_result = await tool.run(image_data=base64.b64encode(image).decode("ascii"))
raw_payload = json.loads(raw_result)
encoded_payload = json.loads(encoded_result)
assert raw_payload["source_type"] == "content"
assert encoded_payload["source_type"] == "content"
assert base64.b64decode(raw_payload["image_base64"]) == image
assert base64.b64decode(encoded_payload["image_base64"]) == image
@pytest.mark.asyncio
async def test_remote_image_uses_ssrf_validation_and_streaming_download(monkeypatch: pytest.MonkeyPatch) -> None:
"""远程图片必须先通过公网校验，再使用有大小上限的异步流下载。"""
@@ -144,6 +162,7 @@ async def test_remote_image_uses_ssrf_validation_and_streaming_download(monkeypa
@pytest.mark.asyncio
async def test_remote_private_or_rejected_url_never_reaches_http(monkeypatch: pytest.MonkeyPatch) -> None:
"""未通过 URL 安全校验时必须在网络边界前失败。"""
async def unsafe_url(*_args: Any, **_kwargs: Any) -> bool:
"""拒绝模拟的内部目标。"""
return False
@@ -165,6 +184,7 @@ async def test_remote_private_or_rejected_url_never_reaches_http(monkeypatch: py
@pytest.mark.asyncio
async def test_remote_content_length_limit_is_enforced_before_body_read(monkeypatch: pytest.MonkeyPatch) -> None:
"""远程响应声明超限时应立即失败，不把大响应交给图片解码器。"""
async def safe_url(*_args: Any, **_kwargs: Any) -> bool:
"""放行测试 URL。"""
return True
@@ -185,9 +205,6 @@ async def test_remote_content_length_limit_is_enforced_before_body_read(monkeypa
def test_image_tool_is_registered_for_agent_but_keeps_raw_data_out_of_generic_formatter() -> None:
"""工具工厂应注册图片能力，成功图片由专用 formatter 保持图块而非文本截断。"""
tool_names = {
tool_class.model_fields["name"].default
for tool_class in MoviePilotToolFactory.BUILTIN_TOOL_CLASSES
}
tool_names = {tool_class.model_fields["name"].default for tool_class in MoviePilotToolFactory.BUILTIN_TOOL_CLASSES}
assert "view_image" in tool_names
assert IMAGE_MAX_BYTES == 768 * 1024