perf(monitor): index incremental snapshot reconciliation

app/modules/filemanager/storages/__init__.py

@@ -330,6 +330,37 @@ class StorageBase(metaclass=ABCMeta):
             if PurePosixPath(file_path).is_relative_to(root_path)
         }
 
+        # 每次目录列举只需要核对该目录的直接子项。旧实现会在每层目录
+        # 上重新扫描整份 files_info，云盘监控目录较多时会退化为
+        # O(目录数 * 文件数)。预先按“目录 -> 直接子项”建立索引，删除
+        # 目录时沿索引移除其完整子树，但不再重复遍历全量快照。
+        previous_children: Dict[PurePosixPath, set[PurePosixPath]] = {}
+        previous_files: Dict[PurePosixPath, str] = {}
+        if files_info:
+            for file_path in files_info:
+                file_posix_path = PurePosixPath(file_path)
+                previous_files[file_posix_path] = file_path
+                try:
+                    relative_parts = file_posix_path.relative_to(root_path).parts
+                except ValueError:
+                    # files_info 已经按 root_path 过滤；保留防御性判断，避免
+                    # 非标准路径对象让增量索引中断整轮监控。
+                    continue
+                for depth in range(len(relative_parts)):
+                    directory_path = root_path.joinpath(*relative_parts[:depth])
+                    child_path = directory_path / relative_parts[depth]
+                    previous_children.setdefault(directory_path, set()).add(child_path)
+
+        def __remove_previous_subtree(path: PurePosixPath) -> None:
+            """沿旧快照索引移除一个已经消失的直接子树。"""
+            pending = [path]
+            while pending:
+                current_path = pending.pop()
+                old_file_path = previous_files.pop(current_path, None)
+                if old_file_path is not None:
+                    files_info.pop(old_file_path, None)
+                pending.extend(previous_children.pop(current_path, ()))
+
         def __remove_deleted_children(_fileitm: _SchemaFileItem,
                                       sub_files: List[_SchemaFileItem]) -> None:
             """
@@ -338,16 +369,9 @@ class StorageBase(metaclass=ABCMeta):
             """
             directory_path = PurePosixPath(_fileitm.path)
             child_paths = {PurePosixPath(sub_file.path) for sub_file in sub_files}
-            for old_file_path in list(files_info):
-                try:
-                    relative_path = PurePosixPath(old_file_path).relative_to(directory_path)
-                except ValueError:
-                    continue
-                if not relative_path.parts:
-                    continue
-                direct_child_path = directory_path / relative_path.parts[0]
+            for direct_child_path in previous_children.pop(directory_path, ()):
                 if direct_child_path not in child_paths:
-                    files_info.pop(old_file_path, None)
+                    __remove_previous_subtree(direct_child_path)
 
         def __snapshot_file(_fileitm: _SchemaFileItem, current_depth: int = 0):
             """

tests/test_snapshot_reconciliation_performance.py

@@ -0,0 +1,90 @@
+from pathlib import Path
+from unittest.mock import MagicMock
+
+from app import schemas
+from app.modules.filemanager.storages import StorageBase
+
+
+def _directory(path: str, *, modify_time: int = 200):
+    return schemas.FileItem(
+        storage="alist",
+        type="dir",
+        path=f"{path.rstrip('/')}/",
+        name=Path(path).name,
+        modify_time=modify_time,
+    )
+
+
+def _file(path: str, *, size: int = 1):
+    return schemas.FileItem(
+        storage="alist",
+        type="file",
+        path=path,
+        name=Path(path).name,
+        size=size,
+        modify_time=200,
+    )
+
+
+def test_snapshot_removes_a_deleted_directory_without_scanning_other_subtrees():
+    """
+    目录删除只应移除该目录的文件，而不影响其它目录的旧基线。
+    """
+    storage = MagicMock()
+    storage.snapshot_check_folder_modtime = True
+    storage.snapshot_strict_query = False
+    storage._snapshot_list = lambda item: StorageBase._snapshot_list(storage, item)
+    root = _directory("/mon")
+    keep = _directory("/mon/keep", modify_time=50)
+    storage.get_item.return_value = root
+    storage.list.return_value = [keep]
+
+    previous_snapshot = {
+        "/mon/gone/episode-1.mkv": {"size": 1, "modify_time": 100},
+        "/mon/gone/episode-2.mkv": {"size": 2, "modify_time": 100},
+        "/mon/keep/episode-1.mkv": {"size": 3, "modify_time": 50},
+    }
+
+    snapshot = StorageBase.snapshot(
+        storage,
+        Path("/mon"),
+        last_snapshot_time=100,
+        previous_snapshot=previous_snapshot,
+    )
+
+    assert snapshot == {
+        "/mon/keep/episode-1.mkv": {"size": 3, "modify_time": 50}
+    }
+
+
+def test_snapshot_reconciles_nested_deleted_files_with_bounded_directory_work():
+    """
+    每个被列举目录只需处理自己的直接子项；嵌套文件删除仍应正确收敛。
+    """
+    storage = MagicMock()
+    storage.snapshot_check_folder_modtime = False
+    storage.snapshot_strict_query = False
+    storage._snapshot_list = lambda item: StorageBase._snapshot_list(storage, item)
+    root = _directory("/mon")
+    season = _directory("/mon/show/Season 1")
+    storage.get_item.return_value = root
+    storage.list.side_effect = [
+        [_directory("/mon/show")],
+        [season],
+        [_file("/mon/show/Season 1/episode-2.mkv", size=20)],
+    ]
+
+    previous_snapshot = {
+        "/mon/show/Season 1/episode-1.mkv": {"size": 10, "modify_time": 100},
+        "/mon/show/Season 1/episode-2.mkv": {"size": 20, "modify_time": 100},
+    }
+
+    snapshot = StorageBase.snapshot(
+        storage,
+        Path("/mon"),
+        previous_snapshot=previous_snapshot,
+    )
+
+    assert snapshot == {
+        "/mon/show/Season 1/episode-2.mkv": {"size": 20, "modify_time": 200, "fileid": None, "type": "file"}
+    }