diff --git a/app/agent/history/schema.py b/app/agent/history/schema.py index 6a1caffb1..5f28da812 100644 --- a/app/agent/history/schema.py +++ b/app/agent/history/schema.py @@ -1,8 +1,10 @@ """独立 state.db 模式与 FTS5 投影，采用 external-content 布局。""" import sqlite3 +import sys from pathlib import Path +BUNDLED_CJK_EXTENSION = Path(sys.prefix) / "lib/moviepilot/libfts5_cjk.so" SCHEMA_VERSION = 1 TOOL_PREFIX_CHARS = 8192 SCHEMA_SQL = """ @@ -39,17 +41,21 @@ INDEXES = {"messages_fts": "unicode61", "messages_fts_trigram": "trigram", "mess def load_cjk(connection: sqlite3.Connection, extension: Path) -> bool: - """只加载宿主固定目录的 CJK tokenizer，加载结束立即关闭扩展权限。""" - if not extension.is_file() or not hasattr(connection, "enable_load_extension"): + """优先使用运行环境自带扩展，兼容旧配置目录；每次加载后立即关闭权限。""" + if not hasattr(connection, "enable_load_extension"): return False - try: - connection.enable_load_extension(True) - connection.load_extension(str(extension)) - return True - except (sqlite3.Error, OSError): - return False - finally: - connection.enable_load_extension(False) + for candidate in (BUNDLED_CJK_EXTENSION, extension): + if not candidate.is_file(): + continue + try: + connection.enable_load_extension(True) + connection.load_extension(str(candidate)) + return True + except (sqlite3.Error, OSError): + continue + finally: + connection.enable_load_extension(False) + return False def index_sql(table: str) -> str: diff --git a/docker/Dockerfile b/docker/Dockerfile index e6d31158b..d3b0a9ec5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -27,6 +27,16 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* +# 按镜像目标架构编译中文索引扩展，编译器不进入最终镜像。 +FROM base AS prepare_cjk + +RUN apt-get update \ + && apt-get install -y --no-install-recommends build-essential \ + && rm -rf /var/lib/apt/lists/* +COPY native/fts5_cjk/ /build/fts5_cjk/ +RUN python3 /build/fts5_cjk/install.py --output /native/libfts5_cjk.so + + # 准备软件包 FROM base AS prepare_package @@ -180,6 +190,11 @@ RUN "${VENV_PATH}/bin/python" /tmp/moviepilot-runtime-dependencies.py --full ARG MOVIEPILOT_PYTHON_VARIANT FROM verify_venv_${MOVIEPILOT_PYTHON_VARIANT} AS prepare_venv +COPY --from=prepare_cjk /native/ ${VENV_PATH}/lib/moviepilot/ +COPY --from=prepare_cjk /build/fts5_cjk/install.py /tmp/verify-cjk.py +RUN "${VENV_PATH}/bin/python" /tmp/verify-cjk.py --check \ + && rm /tmp/verify-cjk.py + # 准备后端源码 FROM base AS prepare_backend diff --git a/docs/agent-history.md b/docs/agent-history.md index d2295f814..9f26a7e32 100644 --- a/docs/agent-history.md +++ b/docs/agent-history.md @@ -14,7 +14,7 @@ | 外部内容 FTS5、触发器原子同步 | 正文只保存一次，FTS 读取固定视图；正文与索引在同一事务提交 | | 标准工具正文前缀 8192、工具 JSON 独立列 | 标准索引保持同一投影；显式 `role_filter="tool"` 搜原始完整工具正文 | | trigram 排除 tool、cron、subagent 及工具调用 JSON | 相同索引范围，减少大结果和重复自动任务的索引体积 | -| `native/fts5_cjk` 的 `cjk_unicode61` | 原版 C 源码随仓库提供，附固定来源和 MIT 许可；可选安装，不用自行生成 n-gram 替代分词语义 | +| `native/fts5_cjk` 的 `cjk_unicode61` | 原版 C 源码随仓库提供，附固定来源和 MIT 许可；Docker 构建与 CLI 安装接入，不用自行生成 n-gram 替代分词语义 | | `hermes_state_search.py`：BM25、短语/布尔/前缀、子串及 OR 重试 | `agent/history/query.py` / `search.py`：普通词首先 FTS5；无结果时子串索引重试，最后允许非 CJK 多词查询 OR 放宽；显式 OR/NOT 不放宽 | | CJK → bigram → trigram → LIKE | 两字中文在可选 tokenizer 可用时走 bigram，三字可走 trigram；单字、缺失索引和显式工具全文回退，响应报告实际路径 | | `tools/session_search_tool.py`：300 候选、默认3/最多10会话、谱系去重 | 相同预算；隐藏 subagent/tool/kanban，cron 排在交互会话后；排除已看谱系；标题优先尝试最新同名续篇 | @@ -36,10 +36,14 @@ ## 中文分词器 -标准 FTS5 和 trigram 来自 SQLite。可选 CJK tokenizer 源码在 `native/fts5_cjk/`，与 Hermes 的可选本地扩展机制一致。按运行环境架构编译并安装到固定目录 `config/agent/runtime/lib/libfts5_cjk.so`，具体命令见该目录 README；Agent 请求不会调用编译器。每次连接只临时开启扩展加载，加载后立即关闭。 +标准 FTS5 和 trigram 来自 SQLite。CJK tokenizer 源码在 `native/fts5_cjk/`。Docker 构建按目标架构编译，使用实际运行时 Python 验证后安装到 `/opt/venv/lib/moviepilot/libfts5_cjk.so`，不会被 `/config` 挂载遮住。CLI 的 `setup`、`install deps`、后端更新会使用所选 venv 编译和验证，同源码、架构且加载验收通过时复用已有文件。运行时优先加载 `/lib/moviepilot/libfts5_cjk.so`，失败或不存在时兼容旧位置 `config/agent/runtime/lib/libfts5_cjk.so`。Agent 请求不会调用编译器，每次连接只临时开启扩展加载，加载后立即关闭。 + +已有 CLI 安装可以运行 `moviepilot install cjk --venv /path/to/venv` 补装，追加 `--check` 只验证。缺少编译器或 SQLite 不支持扩展时，自动安装会明确告警；显式补装/检查返回非零退出码。Docker 旧镜像需要更换为包含本扩展的新构建，单纯更新后端源码不会补出本地库。编译器要求和 Docker 检查命令见 [安装说明](../native/fts5_cjk/README.md)。 没有 CJK 扩展时，两字中文会走 LIKE；这与 Hermes 的能力回退一致，但不等于索引速度。SQLite VM 查询期限限制扫描，超时返回 `query_timeout`/未完成，不能解读为“没有相关内容”。安装扩展后的旧消息在后台分批建索引，期间继续报告索引未就绪。不要跨平台复制已编译二进制。 +扩展安装验收使用内存库实际索引“电影订阅成功”并查询“订阅”；这只证明运行环境能力。各用户历史索引完成后，`session_search` 返回 `index_status.messages_fts_cjk=true`，符合条件的两字查询返回 `search_path="cjk"`，才说明该次查询确实使用了中文索引。 + ## 能力边界和验证 FTS5 是词项检索，不保证任意语义改写都能命中。模型应依据结果用同义词、对象编号、OR、时间区间和已看会话排除继续查询，再用锚点上下文核实。用户明确提供的 URL、文件或实时系统仍然优先读取，不能仅凭历史无命中判断目标不存在。完整存储也不等于完整展示，不能用被截断的单条证据断言工具成功。 diff --git a/docs/cli.md b/docs/cli.md index c74b48681..2bc7b0115 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -14,6 +14,7 @@ curl -fsSL https://raw.githubusercontent.com/jxxghp/MoviePilot/v3/scripts/bootst - 自动检查并尽量安装 `git`、`curl`、`uv 0.12.5+` 和 `Python 3.14+` - 克隆 `MoviePilot` - 安装后端依赖 +- 编译并验证 Agent 中文全文索引扩展；缺少编译器或 SQLite 扩展能力时明确告警并保留检索回退 - 按当前仓库 `version.py` 中的 `FRONTEND_VERSION` 下载对应前端 release 的 `dist.zip` - 下载 `MoviePilot-Resources` 主分支资源 - 将 `resources.v3/*` 同步到后端 `app/application/site/` @@ -127,6 +128,7 @@ moviepilot tool run moviepilot_api operation_id=scheduler.run 'query={"job_id":" ```text moviepilot install deps +moviepilot install cjk moviepilot install frontend moviepilot install resources moviepilot init @@ -167,6 +169,23 @@ moviepilot commands ## 安装命令 +Agent 中文全文索引扩展会随 `install deps`、`setup` 和后端更新安装到所选 venv 的 +`lib/moviepilot/libfts5_cjk.so`。重复安装先校验源码、架构和实际加载能力，无变化时不重复编译。 +一键安装会尝试准备 Linux C 编译器；手动安装需要预先准备 `build-essential`（Debian/Ubuntu）、 +Xcode Command Line Tools（macOS）或 MSVC/MinGW（Windows），且 Python 的 SQLite 支持扩展加载。 + +已有环境可以单独补装或检查，不必重新安装所有依赖： + +```shell +moviepilot install cjk +moviepilot install cjk --venv /path/to/venv +moviepilot install cjk --venv /path/to/venv --check +``` + +检查会在内存库中验证“订阅”两字搜索。失败返回非零退出码并显示原因；自动依赖安装中的 +扩展失败只告警，保留 trigram/LIKE 回退。扩展就绪后历史消息在后台分批建索引， +实际 `session_search` 的 `search_path="cjk"` 表示该次查询已使用中文索引。 + 安装后端依赖： ```shell diff --git a/docs/development-setup.md b/docs/development-setup.md index 949701be0..db73f9fa6 100644 --- a/docs/development-setup.md +++ b/docs/development-setup.md @@ -257,7 +257,7 @@ CI 上传 `complexity-report` 工件，门禁失败时也保留已生成的指 ### Agent 历史消息索引验证 -独立消息库位于 Agent 运行目录，不参与主库 Alembic。使用临时运行目录运行 `tests/test_agent_recall_persistence.py`，不得用真实用户的 `state.db` 做索引破坏或重建测试。可选 CJK tokenizer 保留 Hermes 原版源码及许可证于 `native/fts5_cjk/`；构建命令见该目录 README。测试有本地 C 编译器时在临时目录构建，无编译器时只跳过原生扩展用例，不能宣称该分词路径已验证。常规 FTS5 与 trigram、短中文回退无需该扩展，扩展缺失或失效时索引状态会明确降级。独立 schema 版本不等于产品版本；后续新增主库升级脚本从 3.1.0 命名。 +独立消息库位于 Agent 运行目录，不参与主库 Alembic。使用临时运行目录运行 `tests/test_agent_recall_persistence.py`，不得用真实用户的 `state.db` 做索引破坏或重建测试。CJK tokenizer 保留 Hermes 原版源码及许可证于 `native/fts5_cjk/`；Docker 构建与 CLI 依赖安装使用该目录的 `install.py`，按本机架构编译并用目标解释器实际验证两字中文查询，安装到 `/lib/moviepilot/libfts5_cjk.so`。旧配置目录内的手工扩展作为加载回退；编译器要求和补装/检查命令见该目录 README。测试有本地 C 编译器时在临时目录构建，无编译器时只跳过原生扩展用例，不能宣称该分词路径已验证。常规 FTS5 与 trigram、短中文回退无需该扩展，扩展缺失或失效时索引状态会明确降级。独立 schema 版本不等于产品版本；后续新增主库升级脚本从 3.1.0 命名。 ### Agent 学习验证 diff --git a/docs/docker-startup.md b/docs/docker-startup.md index 07155da05..229fe11d9 100644 --- a/docs/docker-startup.md +++ b/docs/docker-startup.md @@ -6,6 +6,18 @@ 本文基于 `v3` 分支 2026-09-07 的实现整理。实际行为以当前源码为准。 +Agent 中文全文索引扩展在镜像的 `prepare_cjk` 阶段按目标架构编译，最终随 Python venv 放在 +`/opt/venv/lib/moviepilot/libfts5_cjk.so`。构建时由所选 standard/free-threaded Python 完成真实 +加载及两字中文查询验收；运行容器不需要编译器，挂载 `/config` 不影响扩展可见性。 +旧镜像需要重新构建或更换镜像，源码热更新不会自动安装该原生文件。 + +```shell +docker exec /opt/venv/bin/python /app/native/fts5_cjk/install.py --check +``` + +该命令只验证环境能力，不访问用户库。旧历史的 CJK 索引随后分批回填，检索结果通过 +`index_status.messages_fts_cjk` 和 `search_path` 报告是否已经就绪及实际使用的路径。 + ## 1. 文件职责 | 文件 | 运行位置与职责 | diff --git a/moviepilot b/moviepilot index 23eebc211..3a416b8c3 100755 --- a/moviepilot +++ b/moviepilot @@ -10,6 +10,7 @@ Usage: moviepilot [BOOTSTRAP COMMAND] | [RUNTIME COMMAND] Bootstrap Commands: moviepilot install deps [--python PYTHON] [--venv PATH] [--recreate] [--config-dir PATH] + moviepilot install cjk [--venv PATH] [--check] moviepilot install frontend [--version TAG] [--node-version 20.12.1] [--config-dir PATH] moviepilot install resources [--resources-repo PATH] [--resource-dir PATH] [--config-dir PATH] moviepilot init [--skip-resources] [--force-token] [--wizard] [--superuser NAME] [--superuser-password PASSWORD] [--config-dir PATH] @@ -56,6 +57,7 @@ show_commands() { cat <<'EOF' Bootstrap Commands install deps + install cjk install frontend install resources init @@ -99,6 +101,7 @@ show_install_help() { cat <<'EOF' Usage: moviepilot install deps [OPTIONS] + moviepilot install cjk [OPTIONS] moviepilot install frontend [OPTIONS] moviepilot install resources [OPTIONS] @@ -109,6 +112,11 @@ Options: --recreate 删除并重建虚拟环境 --config-dir PATH 指定配置目录 说明 Rust 加速扩展通过 moviepilot-rust PyPI 依赖安装 + 同时尝试编译并验证 Agent 中文全文索引扩展 + + cjk: + --venv PATH 目标虚拟环境目录，默认 ./venv + --check 只验证已安装扩展，不编译或修改文件 frontend: --version TAG 前端版本，默认使用 version.py 中的 FRONTEND_VERSION @@ -473,7 +481,7 @@ show_command_help() { install) shift case "${1:-}" in - ""|deps|-h|--help) + ""|deps|cjk|-h|--help) show_install_help exit 0 ;; @@ -486,7 +494,7 @@ show_command_help() { exit 0 ;; *) - echo "仅支持：moviepilot help install、moviepilot help install deps、moviepilot help install frontend、moviepilot help install resources" >&2 + echo "仅支持：moviepilot help install、moviepilot help install deps、moviepilot help install cjk、moviepilot help install frontend、moviepilot help install resources" >&2 exit 1 ;; esac @@ -602,13 +610,18 @@ case "${1:-}" in require_bootstrap_python exec "$BOOTSTRAP_PYTHON" "$SETUP_SCRIPT" install-frontend "$@" ;; + cjk) + shift + require_bootstrap_python + exec "$BOOTSTRAP_PYTHON" "$SETUP_SCRIPT" install-cjk "$@" + ;; resources) shift require_bootstrap_python exec "$BOOTSTRAP_PYTHON" "$SETUP_SCRIPT" install-resources "$@" ;; *) - echo "支持的命令：moviepilot install deps|frontend|resources" >&2 + echo "支持的命令：moviepilot install deps|cjk|frontend|resources" >&2 exit 1 ;; esac diff --git a/native/fts5_cjk/README.md b/native/fts5_cjk/README.md index 153b679f2..063609038 100644 --- a/native/fts5_cjk/README.md +++ b/native/fts5_cjk/README.md @@ -6,15 +6,52 @@ Author: Soju06 (upstream PR #65544). MIT license: LICENSE.hermes. SQLite headers in vendor/ are public domain, as stated in those files. Trailing whitespace in sqlite3ext.h is normalized; tokenizer code is unchanged. -The tokenizer wraps unicode61 and emits overlapping CJK bigrams. It is -optional exactly as in Hermes; without it, queries use trigram or a timed -LIKE fallback. Compile for the host architecture (never copy binaries -across architectures) and install under the Agent runtime lib directory: +The tokenizer wraps unicode61 and emits overlapping CJK bigrams. Without +it, queries use trigram or a timed LIKE fallback. The C source is unchanged; +`install.py` is MoviePilot's installation and verification entrypoint. + +## Automatic installation + +- Docker builds compile for the image's target architecture in `prepare_cjk`. + The selected runtime Python (standard or free-threaded) verifies the extension + before it is included at `/opt/venv/lib/moviepilot/libfts5_cjk.so`. No compiler + is added to the final image, and `/config` mounts do not hide the library. +- CLI `setup`, `install deps`, and backend updates compile and verify it using + the selected virtual environment. An unchanged, working build is reused. + Bootstrap installs the Linux C toolchain when necessary. Missing compilers or + incompatible SQLite builds produce an explicit warning and retain search fallback. + +To install or check an existing CLI environment: ```bash -cc -shared -fPIC -O2 -Wall -Wextra -Inative/fts5_cjk/vendor \ - native/fts5_cjk/fts5_cjk.c -o /path/to/config/agent/runtime/lib/libfts5_cjk.so +moviepilot install cjk --venv /path/to/venv +moviepilot install cjk --venv /path/to/venv --check ``` -Only this fixed local path is loaded; tools cannot provide extension paths. -No compiler is invoked by an Agent request. +Debian/Ubuntu needs `build-essential`; macOS needs Xcode Command Line Tools +(`xcode-select --install`); Windows needs a matching MSVC developer terminal or +MinGW compiler. `CC` can select the compiler. Python's SQLite must support both +FTS5 and extension loading. Successful verification actually indexes +`电影订阅成功` and matches `订阅` in a temporary in-memory database. + +To check a newly built Docker image/container: + +```bash +docker exec /opt/venv/bin/python /app/native/fts5_cjk/install.py --check +``` + +Existing images need to be rebuilt/replaced to receive the packaged extension; +updating Python source alone does not install a native library in an old image. + +## Runtime loading + +The loader first tries `/lib/moviepilot/libfts5_cjk.so`, then the +legacy `/agent/runtime/lib/libfts5_cjk.so` for manual installations. +Only these fixed local paths are loaded; tools cannot provide extension paths. +No compiler is invoked by an Agent request. Never copy binaries across operating +systems or architectures. Configuration backups do not need to contain the library. + +Older messages are indexed in background batches after installation. A completed +history query reports `index_status.messages_fts_cjk=true`; eligible two-character +queries report `search_path="cjk"`. Installation verification does not claim that +every user's historical index has already finished building. diff --git a/native/fts5_cjk/install.py b/native/fts5_cjk/install.py new file mode 100644 index 000000000..451250219 --- /dev/null +++ b/native/fts5_cjk/install.py @@ -0,0 +1,113 @@ +"""安装阶段编译并验收 CJK 扩展；不导入应用，也不访问用户数据库。""" + +import argparse +import hashlib +import os +import platform +import shlex +import shutil +import sqlite3 +import subprocess +import sys +from pathlib import Path +from tempfile import TemporaryDirectory + +SOURCE = Path(__file__).resolve().parent +LIBRARY = "libfts5_cjk.so" + + +def verify(extension: Path) -> None: + """用当前 Python 的 SQLite 验证动态加载和两字中文子串索引。""" + with sqlite3.connect(":memory:") as connection: + if not hasattr(connection, "enable_load_extension"): + raise RuntimeError("当前 Python 的 SQLite 不支持加载扩展，请更换支持此能力的 Python") + try: + connection.enable_load_extension(True) + connection.load_extension(str(extension)) + finally: + connection.enable_load_extension(False) + connection.execute("CREATE VIRTUAL TABLE probe USING fts5(content, tokenize='cjk_unicode61')") + connection.execute("INSERT INTO probe VALUES ('电影订阅成功')") + count = connection.execute("SELECT count(*) FROM probe WHERE probe MATCH '订阅'").fetchone()[0] + if count != 1: + raise RuntimeError("CJK 扩展已加载，但两字中文索引验收失败") + + +def fingerprint() -> str: + """源码、构建逻辑或目标架构改变时重建，不因重复安装而重复编译。""" + digest = hashlib.sha256(f"{sys.platform}:{platform.machine()}:{sys.maxsize}".encode()) + for name in ("install.py", "fts5_cjk.c", "vendor/sqlite3.h", "vendor/sqlite3ext.h"): + digest.update((SOURCE / name).read_bytes()) + return digest.hexdigest() + + +def compiler_command(output: Path) -> list[str]: + """选择本机 C 工具链；Windows 支持开发者终端中的 MSVC 或 MinGW。""" + configured = os.environ.get("CC", "").strip() + if configured: + compiler = [configured] if Path(configured).is_file() else shlex.split(configured) + else: + available = next((path for name in ("cc", "gcc", "clang", "cl") if (path := shutil.which(name))), None) + if not available: + raise RuntimeError( + "未找到 C 编译器：Debian/Ubuntu 安装 build-essential；macOS 执行 xcode-select --install；" + "Windows 使用 MSVC 开发者终端或 MinGW，再运行 moviepilot install cjk" + ) + compiler = [available] + source = str(SOURCE / "fts5_cjk.c") + headers = str(SOURCE / "vendor") + if Path(compiler[0]).name.lower() in {"cl", "cl.exe"}: + return [*compiler, "/nologo", "/LD", "/O2", f"/I{headers}", source, f"/Fe:{output}", + f"/Fo:{output.with_suffix('.obj')}"] + flags = ["-dynamiclib"] if sys.platform == "darwin" else ["-shared"] + if sys.platform != "win32": + flags.append("-fPIC") + return [*compiler, *flags, "-O2", "-Wall", "-Wextra", f"-I{headers}", source, "-o", str(output)] + + +def install(extension: Path) -> None: + """临时编译、真实加载验收后原子替换，失败时保留原有扩展。""" + expected = fingerprint() + stamp = extension.with_suffix(".sha256") + if extension.is_file() and stamp.is_file() and stamp.read_text(encoding="utf-8").strip() == expected: + try: + verify(extension) + return + except (sqlite3.Error, OSError, RuntimeError): + pass + extension.parent.mkdir(parents=True, exist_ok=True) + with TemporaryDirectory(prefix=".fts5-cjk-", dir=extension.parent) as temporary: + candidate = Path(temporary) / LIBRARY + subprocess.run(compiler_command(candidate), check=True, capture_output=True, text=True, timeout=120, + cwd=temporary) + verify(candidate) + candidate.chmod(0o644) + candidate.replace(extension) + marker = Path(temporary) / stamp.name + marker.write_text(expected + "\n", encoding="utf-8") + marker.chmod(0o644) + marker.replace(stamp) + + +def main() -> int: + """提供 Docker 和 CLI 共用的安装入口，以及无编译的能力检查。""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, default=Path(sys.prefix) / "lib/moviepilot" / LIBRARY) + parser.add_argument("--check", action="store_true", help="只检查已安装扩展，不编译或修改文件") + args = parser.parse_args() + extension = args.output.expanduser().absolute() + try: + if args.check: + verify(extension) + else: + install(extension) + except (OSError, RuntimeError, sqlite3.Error, subprocess.SubprocessError) as error: + detail = error.stderr.strip() if isinstance(error, subprocess.CalledProcessError) and error.stderr else str(error) + print(f"CJK 扩展不可用：{detail}", file=sys.stderr) + return 1 + print(f"CJK 扩展已通过两字中文索引验证：{extension}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/bootstrap-local.sh b/scripts/bootstrap-local.sh index 896c812a4..511476cb2 100755 --- a/scripts/bootstrap-local.sh +++ b/scripts/bootstrap-local.sh @@ -457,6 +457,24 @@ ensure_python() { fi } +# 一键安装准备本机编译器；扩展失败只降级检索，不阻断主程序安装。 +ensure_cjk_compiler() { + if command -v cc >/dev/null 2>&1 || command -v gcc >/dev/null 2>&1 || command -v clang >/dev/null 2>&1; then + return 0 + fi + case "$PACKAGE_MANAGER" in + apt-get) install_system_packages build-essential ;; + dnf|yum|zypper) install_system_packages gcc glibc-devel ;; + pacman) install_system_packages base-devel ;; + apk) install_system_packages build-base ;; + brew) + echo "中文索引扩展需要 Xcode Command Line Tools，请执行 xcode-select --install 后重试 moviepilot install cjk。" >&2 + return 1 + ;; + *) return 1 ;; + esac +} + ensure_prereqs() { if [[ "$OS_NAME" == "Windows" ]]; then echo "检测到当前环境为 Windows shell，建议改用 WSL、Linux 或 macOS 终端运行。" >&2 @@ -467,6 +485,9 @@ ensure_prereqs() { python_install_hint exit 1 fi + if ! ensure_cjk_compiler; then + echo "C 编译器尚未就绪，Agent 中文检索将暂用回退路径；安装后可用 moviepilot install cjk 补装。" >&2 + fi } prompt_text() { diff --git a/scripts/local_setup.py b/scripts/local_setup.py index cbab2a263..4f725f597 100644 --- a/scripts/local_setup.py +++ b/scripts/local_setup.py @@ -2969,12 +2969,29 @@ def install_deps(*, python_bin: str, venv_dir: Path, recreate: bool) -> Path: else: expose_uv_to_venv(uv_bin, venv_dir) install_browser_runtime(venv_python) + install_cjk(venv_python, required=False) return venv_python finally: if temporary_uv_dir is not None: temporary_uv_dir.cleanup() +def install_cjk(venv_python: Path, *, required: bool = True, check: bool = False) -> None: + """由目标解释器安装和验收扩展；自动安装失败明确告警，单独补装失败返回错误。""" + command = [str(venv_python), str(ROOT / "native/fts5_cjk/install.py")] + if check: + command.append("--check") + try: + result = subprocess.run(command, check=True, capture_output=True, text=True, timeout=180, cwd=ROOT) + print_step(result.stdout.strip()) + except (OSError, subprocess.SubprocessError) as error: + detail = error.stderr.strip() if isinstance(error, subprocess.CalledProcessError) and error.stderr else str(error) + message = f"CJK 扩展安装/检查失败：{detail}" + if required: + raise RuntimeError(message) from error + print_step(f"{message}；历史检索暂用 trigram/LIKE 回退，可运行 moviepilot install cjk 重试") + + def install_browser_runtime(venv_python: Path) -> None: """ 预下载 CloakBrowser 浏览器内核，避免首次仿真登录时才拉取大文件。 @@ -3992,6 +4009,10 @@ def build_parser() -> argparse.ArgumentParser: "--config-dir", help="配置目录，默认使用程序目录外的系统配置目录" ) + cjk_parser = subparsers.add_parser("install-cjk", help="补装并验证 Agent 中文全文索引扩展") + cjk_parser.add_argument("--venv", default=str(ROOT / "venv"), help="目标虚拟环境目录") + cjk_parser.add_argument("--check", action="store_true", help="只验证已安装扩展，不编译或修改文件") + frontend_parser = subparsers.add_parser( "install-frontend", help="下载前端 release 并安装本地运行时" ) @@ -4218,6 +4239,10 @@ def main() -> int: ) try: + if args.command == "install-cjk": + install_cjk(get_venv_python(Path(args.venv).expanduser().absolute()), check=args.check) + return 0 + if args.command == "install-deps": venv_python = install_deps( python_bin=args.python, diff --git a/tests/test_cjk_install.py b/tests/test_cjk_install.py new file mode 100644 index 000000000..199bfdd08 --- /dev/null +++ b/tests/test_cjk_install.py @@ -0,0 +1,171 @@ +"""验证原生扩展安装、目标解释器验收及部署入口，不访问用户数据。""" + +import importlib.util +import shutil +import sqlite3 +import subprocess +import sys +from pathlib import Path + +import pytest + +from app.agent.history import schema +from app.agent.history.storage import SqliteRecallRepository +from app.application.messaging.recall import RecallMessage, RecallQuery, RecallSession + +ROOT = Path(__file__).resolve().parents[1] +INSTALLER = ROOT / "native/fts5_cjk/install.py" + + +def load_script(path: Path): + """独立加载纯标准库脚本，避免安装脚本的全局状态污染其他测试。""" + spec = importlib.util.spec_from_file_location(f"test_{path.stem}", path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture(scope="module") +def installed_extension(tmp_path_factory): + """在临时目录用生产安装入口真实编译一次，不写入测试环境或用户目录。""" + if not any(shutil.which(name) for name in ("cc", "gcc", "clang", "cl")): + pytest.skip("原生安装验收需要本机 C 编译器") + extension = tmp_path_factory.mktemp("cjk-install") / "libfts5_cjk.so" + subprocess.run([sys.executable, str(INSTALLER), "--output", str(extension)], check=True, capture_output=True) + return extension + + +def test_installed_extension_activates_history_and_backfills_old_messages(tmp_path, monkeypatch, installed_extension): + """补装后旧消息完成后台回填，两字搜索切换为 cjk，用户数据仍位于配置目录。""" + bundled = tmp_path / "venv/lib/moviepilot/libfts5_cjk.so" + monkeypatch.setattr(schema, "BUNDLED_CJK_EXTENSION", bundled) + repository = SqliteRecallRepository(tmp_path / "config/agent/runtime") + repository.append("alice", RecallSession("old"), (RecallMessage("m", "user", "电影订阅成功"),)) + assert repository.search("alice", RecallQuery(query="订阅"))["search_path"] == "like" + bundled.parent.mkdir(parents=True) + shutil.copy2(installed_extension, bundled) + while repository.maintain("alice")["pending"]: + pass + result = repository.search("alice", RecallQuery(query="订阅")) + assert result["search_path"] == "cjk" and result["count"] == 1 + assert result["index_status"]["messages_fts_cjk"] + assert repository.database_path("alice").is_relative_to(tmp_path / "config") + + +def test_repeated_install_validates_without_recompiling(monkeypatch, installed_extension): + """同源码和架构重复安装不需要编译器，检查模式同样只读。""" + installer = load_script(INSTALLER) + previous = installed_extension.stat().st_mtime_ns + + def unexpected_compile(_output): + """有有效缓存时调用编译器即为回归。""" + pytest.fail("有效扩展不应重新编译") + + monkeypatch.setattr(installer, "compiler_command", unexpected_compile) + installer.install(installed_extension) + subprocess.run([sys.executable, str(INSTALLER), "--output", str(installed_extension), "--check"], + check=True, capture_output=True) + assert installed_extension.stat().st_mtime_ns == previous + + +def test_cli_installs_into_selected_venv_and_check_is_readonly(tmp_path, installed_extension): + """真实 CLI 在独立 venv 中安装、检查，运行时可从该解释器前缀加载。""" + assert installed_extension.is_file() + venv = tmp_path / "selected-venv" + subprocess.run([sys.executable, "-m", "venv", "--without-pip", str(venv)], check=True, capture_output=True) + setup = ROOT / "scripts/local_setup.py" + command = [sys.executable, str(setup), "install-cjk", "--venv", str(venv)] + result = subprocess.run(command, check=True, capture_output=True, text=True) + extension = venv / "lib/moviepilot/libfts5_cjk.so" + assert extension.is_file() and "两字中文索引验证" in result.stdout + previous = extension.stat().st_mtime_ns + subprocess.run([*command, "--check"], check=True, capture_output=True) + assert extension.stat().st_mtime_ns == previous + python = venv / ("Scripts/python.exe" if sys.platform == "win32" else "bin/python") + probe = ( + "import sqlite3; from pathlib import Path; from app.agent.history.schema import load_cjk; " + "assert load_cjk(sqlite3.connect(':memory:'), Path('missing-legacy-extension'))" + ) + subprocess.run([str(python), "-c", probe], cwd=ROOT, check=True, capture_output=True) + + +def test_failed_rebuild_keeps_working_extension(tmp_path, monkeypatch, installed_extension): + """编译失败不覆盖已安装文件，也不留下临时编译目录。""" + installer = load_script(INSTALLER) + extension = tmp_path / "libfts5_cjk.so" + shutil.copy2(installed_extension, extension) + original = extension.read_bytes() + monkeypatch.setattr(installer, "compiler_command", lambda _output: [sys.executable, "-c", "raise SystemExit(1)"]) + with pytest.raises(subprocess.CalledProcessError): + installer.install(extension) + assert extension.read_bytes() == original + assert not list(tmp_path.glob(".fts5-cjk-*")) + installer.verify(extension) + + +def test_invalid_bundled_extension_falls_back_to_legacy_and_disables_loading(tmp_path, monkeypatch, installed_extension): + """跨架构或损坏的自带文件不屏蔽旧手工安装，扩展加载权限始终关闭。""" + invalid = tmp_path / "libfts5_cjk.so" + invalid.write_bytes(b"invalid library") + monkeypatch.setattr(schema, "BUNDLED_CJK_EXTENSION", invalid) + with sqlite3.connect(":memory:") as connection: + assert schema.load_cjk(connection, installed_extension) + with pytest.raises(sqlite3.OperationalError, match="not authorized"): + connection.load_extension(str(installed_extension)) + + +def test_missing_compiler_reports_repair_command(monkeypatch): + """缺少编译器时明确说明补装方式，不能宣称扩展已启用。""" + installer = load_script(INSTALLER) + monkeypatch.delenv("CC", raising=False) + monkeypatch.setattr(installer.shutil, "which", lambda _name: None) + with pytest.raises(RuntimeError, match="moviepilot install cjk"): + installer.compiler_command(Path("unused")) + + +def test_cli_uses_selected_interpreter_and_distinguishes_optional_failure(tmp_path, monkeypatch, capsys): + """自动安装告警降级，显式补装失败返回错误；两者都使用指定 venv。""" + setup = load_script(ROOT / "scripts/local_setup.py") + python = tmp_path / "custom-venv/bin/python" + calls = [] + + def fail(command, **_kwargs): + """模拟目标解释器缺少扩展加载能力。""" + calls.append(command) + raise subprocess.CalledProcessError(1, command, stderr="SQLite extension loading unavailable") + + monkeypatch.setattr(setup.subprocess, "run", fail) + setup.install_cjk(python, required=False) + assert "回退" in capsys.readouterr().out + with pytest.raises(RuntimeError, match="SQLite extension loading unavailable"): + setup.install_cjk(python, check=True) + assert calls == [[str(python), str(INSTALLER)], [str(python), str(INSTALLER), "--check"]] + + +def test_dependency_install_includes_cjk_and_parser_supports_repair(tmp_path, monkeypatch): + """deps 是 setup 和后端升级的共用入口，每次同步后都检查 CJK 扩展。""" + setup = load_script(ROOT / "scripts/local_setup.py") + calls = [] + monkeypatch.setattr(setup, "ensure_supported_python", lambda _python: None) + monkeypatch.setattr(setup, "require_uv", lambda: tmp_path / "uv") + monkeypatch.setattr(setup, "expose_uv_to_venv", lambda *_args: None) + monkeypatch.setattr(setup, "run", lambda *_args, **_kwargs: None) + monkeypatch.setattr(setup, "install_browser_runtime", lambda _python: None) + monkeypatch.setattr(setup, "install_cjk", lambda python, **kwargs: calls.append((python, kwargs))) + python = setup.install_deps(python_bin=sys.executable, venv_dir=tmp_path / "venv", recreate=False) + assert calls == [(python, {"required": False})] + args = setup.build_parser().parse_args(["install-cjk", "--venv", str(tmp_path), "--check"]) + assert args.venv == str(tmp_path) and args.check + + +def test_docker_packages_extension_outside_config_and_verifies_selected_python(): + """镜像按目标架构构建并随 venv 交付，普通和 free-threaded 环境均需真实验收。""" + dockerfile = (ROOT / "docker/Dockerfile").read_text(encoding="utf-8") + stage = dockerfile.split("FROM base AS prepare_cjk", 1)[1].split("FROM base AS prepare_package", 1)[0] + assert "build-essential" in stage + assert "install.py --output /native/libfts5_cjk.so" in stage + selected = dockerfile.split("FROM verify_venv_${MOVIEPILOT_PYTHON_VARIANT} AS prepare_venv", 1)[1] + assert "COPY --from=prepare_cjk /native/ ${VENV_PATH}/lib/moviepilot/" in selected + assert '"${VENV_PATH}/bin/python" /tmp/verify-cjk.py --check' in selected + assert "/config/" not in stage