From 1965cc4de728979530164efd6bce3a7e12fd3cca Mon Sep 17 00:00:00 2001 From: ranzhigg <126594517+ranzhigg@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:49:12 +0800 Subject: [PATCH] perf(monitor): index incremental snapshot reconciliation --- app/modules/filemanager/storages/__init__.py | 42 +++++++-- ...est_snapshot_reconciliation_performance.py | 90 +++++++++++++++++++ 2 files changed, 123 insertions(+), 9 deletions(-) create mode 100644 tests/test_snapshot_reconciliation_performance.py diff --git a/app/modules/filemanager/storages/__init__.py b/app/modules/filemanager/storages/__init__.py index 64f7bf2f7..78667a7f3 100644 --- a/app/modules/filemanager/storages/__init__.py +++ b/app/modules/filemanager/storages/__init__.py @@ -330,6 +330,37 @@ class StorageBase(metaclass=ABCMeta): if PurePosixPath(file_path).is_relative_to(root_path) } + # 每次目录列举只需要核对该目录的直接子项。旧实现会在每层目录 + # 上重新扫描整份 files_info，云盘监控目录较多时会退化为 + # O(目录数 * 文件数)。预先按“目录 -> 直接子项”建立索引，删除 + # 目录时沿索引移除其完整子树，但不再重复遍历全量快照。 + previous_children: Dict[PurePosixPath, set[PurePosixPath]] = {} + previous_files: Dict[PurePosixPath, str] = {} + if files_info: + for file_path in files_info: + file_posix_path = PurePosixPath(file_path) + previous_files[file_posix_path] = file_path + try: + relative_parts = file_posix_path.relative_to(root_path).parts + except ValueError: + # files_info 已经按 root_path 过滤；保留防御性判断，避免 + # 非标准路径对象让增量索引中断整轮监控。 + continue + for depth in range(len(relative_parts)): + directory_path = root_path.joinpath(*relative_parts[:depth]) + child_path = directory_path / relative_parts[depth] + previous_children.setdefault(directory_path, set()).add(child_path) + + def __remove_previous_subtree(path: PurePosixPath) -> None: + """沿旧快照索引移除一个已经消失的直接子树。""" + pending = [path] + while pending: + current_path = pending.pop() + old_file_path = previous_files.pop(current_path, None) + if old_file_path is not None: + files_info.pop(old_file_path, None) + pending.extend(previous_children.pop(current_path, ())) + def __remove_deleted_children(_fileitm: _SchemaFileItem, sub_files: List[_SchemaFileItem]) -> None: """ @@ -338,16 +369,9 @@ class StorageBase(metaclass=ABCMeta): """ directory_path = PurePosixPath(_fileitm.path) child_paths = {PurePosixPath(sub_file.path) for sub_file in sub_files} - for old_file_path in list(files_info): - try: - relative_path = PurePosixPath(old_file_path).relative_to(directory_path) - except ValueError: - continue - if not relative_path.parts: - continue - direct_child_path = directory_path / relative_path.parts[0] + for direct_child_path in previous_children.pop(directory_path, ()): if direct_child_path not in child_paths: - files_info.pop(old_file_path, None) + __remove_previous_subtree(direct_child_path) def __snapshot_file(_fileitm: _SchemaFileItem, current_depth: int = 0): """ diff --git a/tests/test_snapshot_reconciliation_performance.py b/tests/test_snapshot_reconciliation_performance.py new file mode 100644 index 000000000..9362c99ec --- /dev/null +++ b/tests/test_snapshot_reconciliation_performance.py @@ -0,0 +1,90 @@ +from pathlib import Path +from unittest.mock import MagicMock + +from app import schemas +from app.modules.filemanager.storages import StorageBase + + +def _directory(path: str, *, modify_time: int = 200): + return schemas.FileItem( + storage="alist", + type="dir", + path=f"{path.rstrip('/')}/", + name=Path(path).name, + modify_time=modify_time, + ) + + +def _file(path: str, *, size: int = 1): + return schemas.FileItem( + storage="alist", + type="file", + path=path, + name=Path(path).name, + size=size, + modify_time=200, + ) + + +def test_snapshot_removes_a_deleted_directory_without_scanning_other_subtrees(): + """ + 目录删除只应移除该目录的文件，而不影响其它目录的旧基线。 + """ + storage = MagicMock() + storage.snapshot_check_folder_modtime = True + storage.snapshot_strict_query = False + storage._snapshot_list = lambda item: StorageBase._snapshot_list(storage, item) + root = _directory("/mon") + keep = _directory("/mon/keep", modify_time=50) + storage.get_item.return_value = root + storage.list.return_value = [keep] + + previous_snapshot = { + "/mon/gone/episode-1.mkv": {"size": 1, "modify_time": 100}, + "/mon/gone/episode-2.mkv": {"size": 2, "modify_time": 100}, + "/mon/keep/episode-1.mkv": {"size": 3, "modify_time": 50}, + } + + snapshot = StorageBase.snapshot( + storage, + Path("/mon"), + last_snapshot_time=100, + previous_snapshot=previous_snapshot, + ) + + assert snapshot == { + "/mon/keep/episode-1.mkv": {"size": 3, "modify_time": 50} + } + + +def test_snapshot_reconciles_nested_deleted_files_with_bounded_directory_work(): + """ + 每个被列举目录只需处理自己的直接子项；嵌套文件删除仍应正确收敛。 + """ + storage = MagicMock() + storage.snapshot_check_folder_modtime = False + storage.snapshot_strict_query = False + storage._snapshot_list = lambda item: StorageBase._snapshot_list(storage, item) + root = _directory("/mon") + season = _directory("/mon/show/Season 1") + storage.get_item.return_value = root + storage.list.side_effect = [ + [_directory("/mon/show")], + [season], + [_file("/mon/show/Season 1/episode-2.mkv", size=20)], + ] + + previous_snapshot = { + "/mon/show/Season 1/episode-1.mkv": {"size": 10, "modify_time": 100}, + "/mon/show/Season 1/episode-2.mkv": {"size": 20, "modify_time": 100}, + } + + snapshot = StorageBase.snapshot( + storage, + Path("/mon"), + previous_snapshot=previous_snapshot, + ) + + assert snapshot == { + "/mon/show/Season 1/episode-2.mkv": {"size": 20, "modify_time": 200, "fileid": None, "type": "file"} + }