nas/MoviePilot
1
0
Fork 0
mirror of https://github.com/jxxghp/MoviePilot.git synced 2026-11-01 04:47:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(database): support plugin database backup

Browse Source 
支持插件 SQLite 数据库备份并校验 PostgreSQL schema 权限
This commit is contained in:
InfinityPacer
2026-09-12 20:22:33 +08:00
committed by GitHub
parent 20bf902046 999f9cf963
commit b3f748ffa8
11 changed files with 135 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
app/adapters/system/backup/files.py
View File

@@ -11,7 +11,8 @@ from pathlib import Path
from app.runtime.version import get_app_version
_BACKUP_NAME = re.compile(
r"^(?:moviepilot_(?P<version>v\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?)_)?"
r"^(?:(?P<target>[0-9A-Za-z][0-9A-Za-z_-]*)_"
r"(?P<version>v\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?)_)?"
r"(?P<db_type>sqlite|postgresql)_"
r"(?P<timestamp>\d{8}_\d{6})"
r"(?:_(?P<sequence>\d+))?"
@@ -81,13 +82,19 @@ class BackupFiles:
db_type: str,
created_at: datetime,
suffix: str,
target: str = "moviepilot",
version: str | None = None,
) -> str:
"""生成包含数据库类型和秒级时间的简短可读文件名。"""
"""生成包含目标、版本、数据库类型和时间的可读文件名。"""
timestamp = created_at.strftime("%Y%m%d_%H%M%S")
version = get_app_version().strip()
if _RELEASE_VERSION.fullmatch(version) is None:
raise ValueError("程序版本号无法用于数据库备份命名")
base = f"moviepilot_{version}_{db_type}_{timestamp}"
version = (version or get_app_version()).strip()
if not version.startswith("v"):
version = f"v{version}"
if _RELEASE_VERSION.fullmatch(version) is None or not re.fullmatch(
r"[0-9A-Za-z][0-9A-Za-z_-]*", target
):
raise ValueError("数据库备份目标或版本号无法用于文件命名")
base = f"{target}_{version}_{db_type}_{timestamp}"
candidate = f"{base}{suffix}"
sequence = 1
while (self.root / candidate).exists():
@@ -102,6 +109,11 @@ class BackupFiles:
"""从受管文件名读取数据库类型。"""
return BackupFiles._match(name).group("db_type")
@staticmethod
def target(name: str) -> str:
"""从受管文件名读取备份目标，旧格式归属于 MoviePilot。"""
return BackupFiles._match(name).group("target") or "moviepilot"
@staticmethod
def created_at(name: str) -> datetime:
"""从受管文件名读取本地创建时间。"""

13
app/application/backup.py
View File

@@ -33,6 +33,7 @@ class BackupArtifact:
created_at: datetime
path: Path
size: int
target: str = "moviepilot"
@dataclass(frozen=True, slots=True)
@@ -107,12 +108,17 @@ class BackupArtifactStore(Protocol):
db_type: str,
created_at: datetime,
suffix: str,
target: str = "moviepilot",
version: str | None = None,
) -> str:
"""返回当前根目录中可用的正式制品文件名。"""
def database_type(self, name: str) -> str:
"""从受管文件名读取数据库类型。"""
def target(self, name: str) -> str:
"""从受管文件名读取备份目标。"""
def created_at(self, name: str) -> datetime:
"""从受管文件名读取制品创建时间。"""
@@ -137,11 +143,15 @@ class DatabaseBackupService:
artifact_store_factory: BackupArtifactStoreFactory,
policy_reader: Callable[[], BackupPolicy],
clock: Callable[[], datetime] = datetime.now,
target: str = "moviepilot",
version: str | None = None,
) -> None:
self._backend = backend
self._artifact_store_factory = artifact_store_factory
self._policy_reader = policy_reader
self._clock = clock
self._target = target
self._version = version
self._create_lock = Lock()
def create(self) -> BackupArtifact:
@@ -156,6 +166,8 @@ class DatabaseBackupService:
db_type=self._backend.db_type,
created_at=created_at,
suffix=self._backend.suffix,
target=self._target,
version=self._version,
)
temporary = files.create_temporary(self._backend.suffix)
try:
@@ -263,4 +275,5 @@ class DatabaseBackupService:
created_at=created_at or files.created_at(path.name),
path=path,
size=files.size(path.name),
target=files.target(path.name),
)

9
app/application/database.py
View File

@@ -46,10 +46,12 @@ class DatabaseGovernance:
health: DatabaseHealthService,
cleanup: DataCleanupService,
backup: DatabaseBackupService,
plugin_backup: Callable[[], None] | None = None,
) -> None:
self._health = health
self._cleanup = cleanup
self._backup = backup
self._plugin_backup = plugin_backup
def test(self) -> Optional[str]:
"""探测当前活动数据库。"""
@@ -68,8 +70,11 @@ class DatabaseGovernance:
)
def create_backup(self) -> BackupArtifact:
"""创建一个当前活动数据库的一致备份。"""
return self._backup.create()
"""创建宿主数据库备份，并在可用时创建插件 SQLite 备份。"""
artifact = self._backup.create()
if self._plugin_backup is not None:
self._plugin_backup()
return artifact
def list_backups(self) -> tuple[BackupArtifact, ...]:
"""列出受管数据库备份文件。"""

24
app/db/plugin/registry.py
View File

@@ -22,6 +22,7 @@ from app.runtime.settings import get_runtime_setting
__all__ = [
"destroy_database",
"database_handles",
"ensure_database",
"get_database",
"release_all_databases",
@@ -77,6 +78,17 @@ def _build_handle(plugin_id: str) -> PluginDatabaseHandle:
host_engine = get_engine()
with host_engine.begin() as connection:
connection.execute(text(f'CREATE SCHEMA IF NOT EXISTS "{schema}"'))
allowed = connection.execute(
text(
"SELECT has_schema_privilege(current_user, :schema, 'USAGE') "
"AND has_schema_privilege(current_user, :schema, 'CREATE')"
),
{"schema": schema},
).scalar()
if not allowed:
raise PermissionError(
f"数据库账号无权使用或写入插件 schema：{schema}"
)
engine = host_engine.execution_options(schema_translate_map={None: schema})
# 监听器只挂在派生引擎上：OptionEngine 自带 dispatch，仅单向 _join 宿主引擎已有
# 的监听器，宿主与其它插件的连接不会因此改变解析根
@@ -187,10 +199,18 @@ def ensure_database(
# 插件不为它付出导入代价
from app.db.plugin.migration import run_migrations
run_migrations(get_database(plugin_id), migrations)
handle = get_database(plugin_id)
run_migrations(handle, migrations)
return
if models:
_create_declared_tables(get_database(plugin_id), models)
handle = get_database(plugin_id)
_create_declared_tables(handle, models)
def database_handles() -> tuple[PluginDatabaseHandle, ...]:
"""返回当前已建立的插件数据库句柄快照。"""
with _lock:
return tuple(_handles.values())
def release_database(plugin_id: str) -> None:

34
app/startup/composition/database.py
View File

@@ -20,6 +20,7 @@ from app.application.maintenance import (
DataCleanupService,
read_cleanup_policy,
)
from app.application.plugin.runtime import get_existing_plugin_manager
from app.application.plugin.transaction import (
PluginPersistenceService,
configure_plugin_persistence,
@@ -50,6 +51,7 @@ from app.db.engine import get_engine
from app.db.health import probe_database
from app.db.maintenance import DatabaseCleanupRepository
from app.db.oper.systemconfig import SystemConfigOper
from app.db.plugin.registry import database_handles
from app.db.session import (
SessionFactory,
async_session_scope,
@@ -59,6 +61,7 @@ from app.db.uow import (
reset_transaction_runners,
)
from app.db.worker import DatabaseWorker
from app.runtime.log import logger
from app.runtime.settings import get_runtime_setting
@@ -199,6 +202,36 @@ def build_database_governance() -> DatabaseGovernance:
else:
raise RuntimeError(f"不支持的数据库类型：{dialect}")
def backup_plugin_databases() -> None:
"""为当前已建立的 SQLite 插件库创建独立制品。"""
if dialect != "sqlite":
return
manager = get_existing_plugin_manager()
if manager is None:
return
for handle in database_handles():
if handle.db_path is None:
continue
version = manager.get_local_plugin_version(handle.plugin_id)
if not version:
logger.warning(
"跳过插件 %s 的数据库备份：未找到插件版本",
handle.plugin_id,
)
continue
plugin_service = DatabaseBackupService(
backend=SQLiteBackupBackend(handle.engine),
artifact_store_factory=BackupFiles,
policy_reader=lambda: BackupPolicy(
root=get_runtime_setting("DATABASE_BACKUP_PATH") / "plugins" / handle.plugin_id,
retention_days=get_runtime_setting("DB_BACKUP_RETENTION_DAYS"),
max_count=get_runtime_setting("DB_BACKUP_MAX_COUNT"),
),
target=handle.plugin_id,
version=version,
)
plugin_service.create()
return DatabaseGovernance(
health=DatabaseHealthService(probe_database),
cleanup=DataCleanupService(
@@ -210,6 +243,7 @@ def build_database_governance() -> DatabaseGovernance:
artifact_store_factory=BackupFiles,
policy_reader=read_backup_policy,
),
plugin_backup=backup_plugin_databases,
)

2
docs/architecture-overview.md
View File

@@ -757,7 +757,7 @@ flowchart LR
| 指标 | 当前值 |
|---|---:|
| Python 模块 | 1010 |
| 内部导入边 | 8,581 |
| 内部导入边 | 8,585 |
| 非平凡 SCC | 1（精确 containment 的 TMDB 移植包环） |
| Application / Chain 具体 Adapter 直连 | 0 / 0 |
| Direct egress | 53（债务已清零，53 条精确 containment） |

2
docs/cli.md
View File

@@ -529,6 +529,8 @@ docker compose start <service>
说明：
- SQLite 使用在线备份 API，PostgreSQL 使用镜像内置的 `pg_dump` custom format
- SQLite 插件自有数据库会在创建宿主备份时同步备份到 `database_backup/plugins/<插件ID>/`，文件名格式为 `<插件ID>_<插件版本>_sqlite_<时间>.db`
- PostgreSQL 插件自有数据库使用独立 schema；插件建库会校验当前账号的 schema 使用和建表权限，权限不足时插件不会进入运行态
- 源码部署使用 PostgreSQL 时，宿主机需安装 `pg_dump` 和 `pg_restore` 并加入 `PATH`；Docker 镜像已内置
- 默认目录为配置目录下的 `database_backup/`，可通过 `DB_BACKUP_PATH` 调整
- 备份、列举和校验可独立通过 CLI 执行

2
docs/refactor/optimization-checklist.md
View File

@@ -94,7 +94,7 @@ ARCH-201 至 ARCH-204 均达到实现、验证、提交、推送和远端门禁
| 指标 | 当前值 | 解释 |
|---|---:|---|
| 宿主 Python 模块 / 内部依赖边 | 1010 / 8,581 | `dependency-baseline.json` 当前快照；分类、下载资源归类、订阅搜索、整理恢复、Agent 计划、工具视觉、终端生命周期与终端作用域模块的受控依赖 |
| 宿主 Python 模块 / 内部依赖边 | 1010 / 8,585 | `dependency-baseline.json` 当前快照；分类、下载资源归类、订阅搜索、整理恢复、Agent 计划、工具视觉、终端生命周期与终端作用域模块的受控依赖 |
| 非平凡 SCC | 1 | 仅保留精确 containment 的 29 模块 TMDB 移植包环 |
| 跨层 DB 边界债务 | 0 | Application、Chain、API、Agent、Runtime、Workflow 到 DB 的受控债务均为零 |
| Model/Oper 事务债务 | 0 | 自建 Session、自动事务装饰器、直接 commit/rollback 等基线均为零 |

8
tests/fixtures/architecture/dependency-baseline.json vendored
View File

@@ -1077,8 +1077,8 @@
"runtime_only": true
}
},
"edge_count": 8581,
"edge_sha256": "7b89a2f04c4b256955e9d17821d7e4ac448c1847e12772aadf5906a520a4e3eb",
"edge_count": 8585,
"edge_sha256": "bdb99b1fdd69962aab0c46f7f90d9e9a8400d91710a7baf6261ce999b24795a4",
"edges": [
"app -> app.foundation",
"app -> app.foundation.environment",
@@ -8916,6 +8916,7 @@
"app.startup.composition.database -> app.application.database",
"app.startup.composition.database -> app.application.maintenance",
"app.startup.composition.database -> app.application.plugin",
"app.startup.composition.database -> app.application.plugin.runtime",
"app.startup.composition.database -> app.application.plugin.transaction",
"app.startup.composition.database -> app.application.query",
"app.startup.composition.database -> app.application.workflow",
@@ -8932,10 +8933,13 @@
"app.startup.composition.database -> app.db.maintenance",
"app.startup.composition.database -> app.db.oper",
"app.startup.composition.database -> app.db.oper.systemconfig",
"app.startup.composition.database -> app.db.plugin",
"app.startup.composition.database -> app.db.plugin.registry",
"app.startup.composition.database -> app.db.session",
"app.startup.composition.database -> app.db.uow",
"app.startup.composition.database -> app.db.worker",
"app.startup.composition.database -> app.runtime",
"app.startup.composition.database -> app.runtime.log",
"app.startup.composition.database -> app.runtime.settings",
"app.startup.composition.domain -> app.adapters",
"app.startup.composition.domain -> app.adapters.network",

17
tests/test_database_backup_service.py
View File

@@ -154,6 +154,23 @@ def test_backup_name_uses_application_release_version(
assert _service(tmp_path).create().name == "moviepilot_v4.2.1_sqlite_20260819_134526.db"
def test_backup_name_uses_plugin_target_and_version(tmp_path: Path) -> None:
"""插件备份文件名应记录插件 ID 与插件版本，而不是宿主版本。"""
service = DatabaseBackupService(
backend=_Backend(),
artifact_store_factory=BackupFiles,
policy_reader=lambda: BackupPolicy(tmp_path),
clock=lambda: datetime(2026, 8, 19, 13, 45, 26),
target="DemoPlugin",
version="1.2.3",
)
artifact = service.create()
assert artifact.name == "DemoPlugin_v1.2.3_sqlite_20260819_134526.db"
assert artifact.target == "DemoPlugin"
def test_retention_applies_after_new_artifact_is_available(tmp_path: Path) -> None:
old = _service(tmp_path, now=datetime(2026, 8, 1, 3, 0, 0)).create()
current = _service(

14
tests/test_db_plugin_framework.py
View File

@@ -546,6 +546,20 @@ def test_postgresql_handle_creates_the_plugin_schema(postgresql_backend):
assert f'CREATE SCHEMA IF NOT EXISTS "{handle.schema}"' in executed
def test_postgresql_handle_rejects_schema_without_write_privilege(postgresql_backend):
"""已有 schema 但当前账号不可写时，插件建库必须直接失败。"""
host_engine, _ = postgresql_backend
connection = host_engine.begin.return_value.__enter__.return_value
denied = MagicMock()
denied.scalar.return_value = False
connection.execute.side_effect = [None, denied]
with pytest.raises(PermissionError, match="无权使用或写入插件 schema"):
registry_module.get_database("demo")
assert "demo" not in registry_module._handles
def test_concurrent_get_database_builds_a_single_handle(plugin_data_root, sqlite_backend):
"""八个线程同时取同一插件的句柄时只建出一个句柄，不会并存两份连接池。"""
thread_count = 8