From 1ec7005b51e81c2a6a523bd44c41d5d85fac1525 Mon Sep 17 00:00:00 2001 From: InfinityPacer Date: Sat, 12 Sep 2026 19:53:38 +0800 Subject: [PATCH 1/3] feat(database): back up plugin sqlite databases --- app/adapters/system/backup/files.py | 24 ++++++++++++++----- app/application/backup.py | 8 +++++++ app/application/database.py | 9 +++++-- app/db/plugin/registry.py | 24 +++++++++++++++++-- app/startup/composition/database.py | 34 +++++++++++++++++++++++++++ docs/cli.md | 2 ++ tests/test_database_backup_service.py | 17 ++++++++++++++ tests/test_db_plugin_framework.py | 14 +++++++++++ 8 files changed, 122 insertions(+), 10 deletions(-) diff --git a/app/adapters/system/backup/files.py b/app/adapters/system/backup/files.py index 6711251cf..567d96f8c 100644 --- a/app/adapters/system/backup/files.py +++ b/app/adapters/system/backup/files.py @@ -11,7 +11,8 @@ from pathlib import Path from app.runtime.version import get_app_version _BACKUP_NAME = re.compile( - r"^(?:moviepilot_(?Pv\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?)_)?" + r"^(?:(?P[0-9A-Za-z][0-9A-Za-z_-]*)_" + r"(?Pv\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?)_)?" r"(?Psqlite|postgresql)_" r"(?P\d{8}_\d{6})" r"(?:_(?P\d+))?" @@ -81,13 +82,19 @@ class BackupFiles: db_type: str, created_at: datetime, suffix: str, + target: str = "moviepilot", + version: str | None = None, ) -> str: - """生成包含数据库类型和秒级时间的简短可读文件名。""" + """生成包含目标、版本、数据库类型和时间的可读文件名。""" timestamp = created_at.strftime("%Y%m%d_%H%M%S") - version = get_app_version().strip() - if _RELEASE_VERSION.fullmatch(version) is None: - raise ValueError("程序版本号无法用于数据库备份命名") - base = f"moviepilot_{version}_{db_type}_{timestamp}" + version = (version or get_app_version()).strip() + if not version.startswith("v"): + version = f"v{version}" + if _RELEASE_VERSION.fullmatch(version) is None or not re.fullmatch( + r"[0-9A-Za-z][0-9A-Za-z_-]*", target + ): + raise ValueError("数据库备份目标或版本号无法用于文件命名") + base = f"{target}_{version}_{db_type}_{timestamp}" candidate = f"{base}{suffix}" sequence = 1 while (self.root / candidate).exists(): @@ -102,6 +109,11 @@ class BackupFiles: """从受管文件名读取数据库类型。""" return BackupFiles._match(name).group("db_type") + @staticmethod + def target(name: str) -> str: + """从受管文件名读取备份目标，旧格式归属于 MoviePilot。""" + return BackupFiles._match(name).group("target") or "moviepilot" + @staticmethod def created_at(name: str) -> datetime: """从受管文件名读取本地创建时间。""" diff --git a/app/application/backup.py b/app/application/backup.py index fb286e228..7bbd89951 100644 --- a/app/application/backup.py +++ b/app/application/backup.py @@ -33,6 +33,7 @@ class BackupArtifact: created_at: datetime path: Path size: int + target: str = "moviepilot" @dataclass(frozen=True, slots=True) @@ -137,11 +138,15 @@ class DatabaseBackupService: artifact_store_factory: BackupArtifactStoreFactory, policy_reader: Callable[[], BackupPolicy], clock: Callable[[], datetime] = datetime.now, + target: str = "moviepilot", + version: str | None = None, ) -> None: self._backend = backend self._artifact_store_factory = artifact_store_factory self._policy_reader = policy_reader self._clock = clock + self._target = target + self._version = version self._create_lock = Lock() def create(self) -> BackupArtifact: @@ -156,6 +161,8 @@ class DatabaseBackupService: db_type=self._backend.db_type, created_at=created_at, suffix=self._backend.suffix, + target=self._target, + version=self._version, ) temporary = files.create_temporary(self._backend.suffix) try: @@ -263,4 +270,5 @@ class DatabaseBackupService: created_at=created_at or files.created_at(path.name), path=path, size=files.size(path.name), + target=files.target(path.name), ) diff --git a/app/application/database.py b/app/application/database.py index 03f9b638b..c2e320104 100644 --- a/app/application/database.py +++ b/app/application/database.py @@ -46,10 +46,12 @@ class DatabaseGovernance: health: DatabaseHealthService, cleanup: DataCleanupService, backup: DatabaseBackupService, + plugin_backup: Callable[[], None] | None = None, ) -> None: self._health = health self._cleanup = cleanup self._backup = backup + self._plugin_backup = plugin_backup def test(self) -> Optional[str]: """探测当前活动数据库。""" @@ -68,8 +70,11 @@ class DatabaseGovernance: ) def create_backup(self) -> BackupArtifact: - """创建一个当前活动数据库的一致备份。""" - return self._backup.create() + """创建宿主数据库备份，并在可用时创建插件 SQLite 备份。""" + artifact = self._backup.create() + if self._plugin_backup is not None: + self._plugin_backup() + return artifact def list_backups(self) -> tuple[BackupArtifact, ...]: """列出受管数据库备份文件。""" diff --git a/app/db/plugin/registry.py b/app/db/plugin/registry.py index 73a9e3ef7..f32d0d18e 100644 --- a/app/db/plugin/registry.py +++ b/app/db/plugin/registry.py @@ -22,6 +22,7 @@ from app.runtime.settings import get_runtime_setting __all__ = [ "destroy_database", + "database_handles", "ensure_database", "get_database", "release_all_databases", @@ -77,6 +78,17 @@ def _build_handle(plugin_id: str) -> PluginDatabaseHandle: host_engine = get_engine() with host_engine.begin() as connection: connection.execute(text(f'CREATE SCHEMA IF NOT EXISTS "{schema}"')) + allowed = connection.execute( + text( + "SELECT has_schema_privilege(current_user, :schema, 'USAGE') " + "AND has_schema_privilege(current_user, :schema, 'CREATE')" + ), + {"schema": schema}, + ).scalar() + if not allowed: + raise PermissionError( + f"数据库账号无权使用或写入插件 schema：{schema}" + ) engine = host_engine.execution_options(schema_translate_map={None: schema}) # 监听器只挂在派生引擎上：OptionEngine 自带 dispatch，仅单向 _join 宿主引擎已有 # 的监听器，宿主与其它插件的连接不会因此改变解析根 @@ -187,10 +199,18 @@ def ensure_database( # 插件不为它付出导入代价 from app.db.plugin.migration import run_migrations - run_migrations(get_database(plugin_id), migrations) + handle = get_database(plugin_id) + run_migrations(handle, migrations) return if models: - _create_declared_tables(get_database(plugin_id), models) + handle = get_database(plugin_id) + _create_declared_tables(handle, models) + + +def database_handles() -> tuple[PluginDatabaseHandle, ...]: + """返回当前已建立的插件数据库句柄快照。""" + with _lock: + return tuple(_handles.values()) def release_database(plugin_id: str) -> None: diff --git a/app/startup/composition/database.py b/app/startup/composition/database.py index d696ac0b7..038d1599f 100644 --- a/app/startup/composition/database.py +++ b/app/startup/composition/database.py @@ -20,6 +20,7 @@ from app.application.maintenance import ( DataCleanupService, read_cleanup_policy, ) +from app.application.plugin.runtime import get_existing_plugin_manager from app.application.plugin.transaction import ( PluginPersistenceService, configure_plugin_persistence, @@ -50,6 +51,7 @@ from app.db.engine import get_engine from app.db.health import probe_database from app.db.maintenance import DatabaseCleanupRepository from app.db.oper.systemconfig import SystemConfigOper +from app.db.plugin.registry import database_handles from app.db.session import ( SessionFactory, async_session_scope, @@ -59,6 +61,7 @@ from app.db.uow import ( reset_transaction_runners, ) from app.db.worker import DatabaseWorker +from app.runtime.log import logger from app.runtime.settings import get_runtime_setting @@ -199,6 +202,36 @@ def build_database_governance() -> DatabaseGovernance: else: raise RuntimeError(f"不支持的数据库类型：{dialect}") + def backup_plugin_databases() -> None: + """为当前已建立的 SQLite 插件库创建独立制品。""" + if dialect != "sqlite": + return + manager = get_existing_plugin_manager() + if manager is None: + return + for handle in database_handles(): + if handle.db_path is None: + continue + version = manager.get_local_plugin_version(handle.plugin_id) + if not version: + logger.warning( + "跳过插件 %s 的数据库备份：未找到插件版本", + handle.plugin_id, + ) + continue + plugin_service = DatabaseBackupService( + backend=SQLiteBackupBackend(handle.engine), + artifact_store_factory=BackupFiles, + policy_reader=lambda: BackupPolicy( + root=get_runtime_setting("DATABASE_BACKUP_PATH") / "plugins" / handle.plugin_id, + retention_days=get_runtime_setting("DB_BACKUP_RETENTION_DAYS"), + max_count=get_runtime_setting("DB_BACKUP_MAX_COUNT"), + ), + target=handle.plugin_id, + version=version, + ) + plugin_service.create() + return DatabaseGovernance( health=DatabaseHealthService(probe_database), cleanup=DataCleanupService( @@ -210,6 +243,7 @@ def build_database_governance() -> DatabaseGovernance: artifact_store_factory=BackupFiles, policy_reader=read_backup_policy, ), + plugin_backup=backup_plugin_databases, ) diff --git a/docs/cli.md b/docs/cli.md index 48e27c198..5ce336539 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -529,6 +529,8 @@ docker compose start 说明： - SQLite 使用在线备份 API，PostgreSQL 使用镜像内置的 `pg_dump` custom format +- SQLite 插件自有数据库会在创建宿主备份时同步备份到 `database_backup/plugins/<插件ID>/`，文件名格式为 `<插件ID>_<插件版本>_sqlite_<时间>.db` +- PostgreSQL 插件自有数据库使用独立 schema；插件建库会校验当前账号的 schema 使用和建表权限，权限不足时插件不会进入运行态 - 源码部署使用 PostgreSQL 时，宿主机需安装 `pg_dump` 和 `pg_restore` 并加入 `PATH`；Docker 镜像已内置 - 默认目录为配置目录下的 `database_backup/`，可通过 `DB_BACKUP_PATH` 调整 - 备份、列举和校验可独立通过 CLI 执行 diff --git a/tests/test_database_backup_service.py b/tests/test_database_backup_service.py index fc9eade5f..4de2e55e4 100644 --- a/tests/test_database_backup_service.py +++ b/tests/test_database_backup_service.py @@ -154,6 +154,23 @@ def test_backup_name_uses_application_release_version( assert _service(tmp_path).create().name == "moviepilot_v4.2.1_sqlite_20260819_134526.db" +def test_backup_name_uses_plugin_target_and_version(tmp_path: Path) -> None: + """插件备份文件名应记录插件 ID 与插件版本，而不是宿主版本。""" + service = DatabaseBackupService( + backend=_Backend(), + artifact_store_factory=BackupFiles, + policy_reader=lambda: BackupPolicy(tmp_path), + clock=lambda: datetime(2026, 8, 19, 13, 45, 26), + target="DemoPlugin", + version="1.2.3", + ) + + artifact = service.create() + + assert artifact.name == "DemoPlugin_v1.2.3_sqlite_20260819_134526.db" + assert artifact.target == "DemoPlugin" + + def test_retention_applies_after_new_artifact_is_available(tmp_path: Path) -> None: old = _service(tmp_path, now=datetime(2026, 8, 1, 3, 0, 0)).create() current = _service( diff --git a/tests/test_db_plugin_framework.py b/tests/test_db_plugin_framework.py index 879644d1d..488649456 100644 --- a/tests/test_db_plugin_framework.py +++ b/tests/test_db_plugin_framework.py @@ -546,6 +546,20 @@ def test_postgresql_handle_creates_the_plugin_schema(postgresql_backend): assert f'CREATE SCHEMA IF NOT EXISTS "{handle.schema}"' in executed +def test_postgresql_handle_rejects_schema_without_write_privilege(postgresql_backend): + """已有 schema 但当前账号不可写时，插件建库必须直接失败。""" + host_engine, _ = postgresql_backend + connection = host_engine.begin.return_value.__enter__.return_value + denied = MagicMock() + denied.scalar.return_value = False + connection.execute.side_effect = [None, denied] + + with pytest.raises(PermissionError, match="无权使用或写入插件 schema"): + registry_module.get_database("demo") + + assert "demo" not in registry_module._handles + + def test_concurrent_get_database_builds_a_single_handle(plugin_data_root, sqlite_backend): """八个线程同时取同一插件的句柄时只建出一个句柄，不会并存两份连接池。""" thread_count = 8 From 427271b1acecf3bd61a0cb3e9f7539cc35b031b9 Mon Sep 17 00:00:00 2001 From: InfinityPacer Date: Sat, 12 Sep 2026 20:04:03 +0800 Subject: [PATCH 2/3] chore(architecture): update database backup dependency baseline --- docs/architecture-overview.md | 2 +- docs/refactor/optimization-checklist.md | 2 +- tests/fixtures/architecture/dependency-baseline.json | 8 ++++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/architecture-overview.md b/docs/architecture-overview.md index b2d3d4d96..8d31a7678 100644 --- a/docs/architecture-overview.md +++ b/docs/architecture-overview.md @@ -757,7 +757,7 @@ flowchart LR | 指标 | 当前值 | |---|---:| | Python 模块 | 1010 | -| 内部导入边 | 8,581 | +| 内部导入边 | 8,585 | | 非平凡 SCC | 1（精确 containment 的 TMDB 移植包环） | | Application / Chain 具体 Adapter 直连 | 0 / 0 | | Direct egress | 53（债务已清零，53 条精确 containment） | diff --git a/docs/refactor/optimization-checklist.md b/docs/refactor/optimization-checklist.md index a6bd99d2e..04e32afca 100644 --- a/docs/refactor/optimization-checklist.md +++ b/docs/refactor/optimization-checklist.md @@ -94,7 +94,7 @@ ARCH-201 至 ARCH-204 均达到实现、验证、提交、推送和远端门禁 | 指标 | 当前值 | 解释 | |---|---:|---| -| 宿主 Python 模块 / 内部依赖边 | 1010 / 8,581 | `dependency-baseline.json` 当前快照；分类、下载资源归类、订阅搜索、整理恢复、Agent 计划、工具视觉、终端生命周期与终端作用域模块的受控依赖 | +| 宿主 Python 模块 / 内部依赖边 | 1010 / 8,585 | `dependency-baseline.json` 当前快照；分类、下载资源归类、订阅搜索、整理恢复、Agent 计划、工具视觉、终端生命周期与终端作用域模块的受控依赖 | | 非平凡 SCC | 1 | 仅保留精确 containment 的 29 模块 TMDB 移植包环 | | 跨层 DB 边界债务 | 0 | Application、Chain、API、Agent、Runtime、Workflow 到 DB 的受控债务均为零 | | Model/Oper 事务债务 | 0 | 自建 Session、自动事务装饰器、直接 commit/rollback 等基线均为零 | diff --git a/tests/fixtures/architecture/dependency-baseline.json b/tests/fixtures/architecture/dependency-baseline.json index fadcfa8a4..0c1705b0e 100644 --- a/tests/fixtures/architecture/dependency-baseline.json +++ b/tests/fixtures/architecture/dependency-baseline.json @@ -1077,8 +1077,8 @@ "runtime_only": true } }, - "edge_count": 8581, - "edge_sha256": "7b89a2f04c4b256955e9d17821d7e4ac448c1847e12772aadf5906a520a4e3eb", + "edge_count": 8585, + "edge_sha256": "bdb99b1fdd69962aab0c46f7f90d9e9a8400d91710a7baf6261ce999b24795a4", "edges": [ "app -> app.foundation", "app -> app.foundation.environment", @@ -8916,6 +8916,7 @@ "app.startup.composition.database -> app.application.database", "app.startup.composition.database -> app.application.maintenance", "app.startup.composition.database -> app.application.plugin", + "app.startup.composition.database -> app.application.plugin.runtime", "app.startup.composition.database -> app.application.plugin.transaction", "app.startup.composition.database -> app.application.query", "app.startup.composition.database -> app.application.workflow", @@ -8932,10 +8933,13 @@ "app.startup.composition.database -> app.db.maintenance", "app.startup.composition.database -> app.db.oper", "app.startup.composition.database -> app.db.oper.systemconfig", + "app.startup.composition.database -> app.db.plugin", + "app.startup.composition.database -> app.db.plugin.registry", "app.startup.composition.database -> app.db.session", "app.startup.composition.database -> app.db.uow", "app.startup.composition.database -> app.db.worker", "app.startup.composition.database -> app.runtime", + "app.startup.composition.database -> app.runtime.log", "app.startup.composition.database -> app.runtime.settings", "app.startup.composition.domain -> app.adapters", "app.startup.composition.domain -> app.adapters.network", From 999f9cf963ec23ee38b75272ef485119a55b4e46 Mon Sep 17 00:00:00 2001 From: InfinityPacer Date: Sat, 12 Sep 2026 20:12:27 +0800 Subject: [PATCH 3/3] fix(database): align backup store protocol --- app/application/backup.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/application/backup.py b/app/application/backup.py index 7bbd89951..fed016fdb 100644 --- a/app/application/backup.py +++ b/app/application/backup.py @@ -108,12 +108,17 @@ class BackupArtifactStore(Protocol): db_type: str, created_at: datetime, suffix: str, + target: str = "moviepilot", + version: str | None = None, ) -> str: """返回当前根目录中可用的正式制品文件名。""" def database_type(self, name: str) -> str: """从受管文件名读取数据库类型。""" + def target(self, name: str) -> str: + """从受管文件名读取备份目标。""" + def created_at(self, name: str) -> datetime: """从受管文件名读取制品创建时间。"""