fix(transfer): 修复部分情况下无法正确统计已完成任务总大小的问题

- get_directory_size 使用 os.scandir 递归遍历提升性能
- 当任务文件项存储类型为 local 时,若其大小为空,则通过 SystemUtils 获取目录大小以确保
完成任务的准确统计。

fix(cache): 修改 fresh 和 async_fresh 默认参数为 True

refactor(filemanager): 移除整理后总大小计算逻辑

- 删除 TransHandler 中对整理目录总大小的冗余计算,提升性能并简化流程。

perf(system): 使用 scandir 优化文件扫描性能

- 重构 SystemUtils 中的文件扫描方法(list_files、exists_file、list_sub_files),
- 采用 os.scandir 替代 glob 实现,并预编译正则表达式以提升目录遍历与文件匹配性能。
This commit is contained in:
Attente
2025-10-23 19:21:24 +08:00
parent 0c58eae5e7
commit 7421ca09cc
4 changed files with 88 additions and 49 deletions

View File

@@ -33,6 +33,7 @@ from app.schemas.types import TorrentStatus, EventType, MediaType, ProgressKey,
SystemConfigKey, ChainEventType, ContentType
from app.utils.singleton import Singleton
from app.utils.string import StringUtils
from app.utils.system import SystemUtils
downloader_lock = threading.Lock()
job_lock = threading.Lock()
@@ -329,8 +330,12 @@ class JobManager:
# 计算状态为完成的任务数
if __mediaid__ not in self._job_view:
return 0
return sum([task.fileitem.size for task in self._job_view[__mediaid__].tasks if
task.state == "completed" and task.fileitem.size is not None])
return sum([
task.fileitem.size if task.fileitem.size is not None
else (SystemUtils.get_directory_size(Path(task.fileitem.path)) if task.fileitem.storage == "local" else 0)
for task in self._job_view[__mediaid__].tasks
if task.state == "completed"
])
def total(self) -> int:
"""

View File

@@ -1016,7 +1016,7 @@ class AsyncFileBackend(AsyncCacheBackend):
@contextmanager
def fresh(fresh: bool = False):
def fresh(fresh: bool = True):
"""
是否获取新数据(不使用缓存的值)
@@ -1031,7 +1031,7 @@ def fresh(fresh: bool = False):
_fresh.reset(token)
@asynccontextmanager
async def async_fresh(fresh: bool = False):
async def async_fresh(fresh: bool = True):
"""
是否获取新数据(不使用缓存的值)

View File

@@ -150,14 +150,11 @@ class TransHandler:
return self.result.copy()
logger.info(f"文件夹 {fileitem.path} 整理成功")
# 计算目录下所有文件大小
total_size = sum(file.stat().st_size for file in Path(fileitem.path).rglob('*') if file.is_file())
# 返回整理后的路径
self.__set_result(success=True,
fileitem=fileitem,
target_item=new_diritem,
target_diritem=new_diritem,
total_size=total_size,
need_scrape=need_scrape,
need_notify=need_notify,
transfer_type=transfer_type)

View File

@@ -7,7 +7,6 @@ import shutil
import subprocess
import sys
import uuid
from glob import glob
from pathlib import Path
from typing import List, Optional, Tuple, Union
@@ -225,23 +224,31 @@ class SystemUtils:
if directory.is_file():
return [directory]
if not min_filesize:
min_filesize = 0
files = []
# 预编译正则表达式
if extensions:
pattern = r".*(" + "|".join(extensions) + ")$"
pattern = re.compile(r".*(" + "|".join(extensions) + r")$", re.IGNORECASE)
else:
pattern = r".*"
pattern = re.compile(r".*")
# 遍历目录及子目录
for matched_glob in glob('**', root_dir=directory, recursive=recursive, include_hidden=True):
path = directory.joinpath(matched_glob)
if path.is_file() \
and re.match(pattern, path.name, re.IGNORECASE) \
and path.stat().st_size >= min_filesize * 1024 * 1024:
files.append(path)
def _scan_directory(dir_path: Path, is_recursive: bool):
try:
with os.scandir(dir_path) as entries:
for entry in entries:
try:
if entry.is_file(follow_symlinks=False):
entry_path = Path(entry.path)
if (pattern.match(entry.name) and
(min_filesize <= 0 or entry.stat().st_size >= min_filesize * 1024 * 1024)):
files.append(entry_path)
elif entry.is_dir() and is_recursive:
_scan_directory(Path(entry.path), is_recursive)
except (OSError, PermissionError):
continue
except (OSError, PermissionError):
pass
_scan_directory(directory, recursive)
return files
@staticmethod
@@ -256,29 +263,44 @@ class SystemUtils:
:return: True存在 False不存在
"""
if not min_filesize:
min_filesize = 0
if not directory.exists():
return False
# 预编译正则表达式
if extensions:
pattern = re.compile(r".*(" + "|".join(extensions) + r")$", re.IGNORECASE)
else:
pattern = re.compile(r".*")
if directory.is_file():
# 检查单个文件是否符合条件
if extensions and not pattern.match(directory.name):
return False
if min_filesize > 0 and directory.stat().st_size < min_filesize * 1024 * 1024:
return False
return True
if not min_filesize:
min_filesize = 0
def _search_files(dir_path: Path, is_recursive: bool) -> bool:
try:
with os.scandir(dir_path) as entries:
for entry in entries:
try:
if entry.is_file(follow_symlinks=False):
# 检查文件是否符合条件
if (pattern.match(entry.name) and
(min_filesize <= 0 or entry.stat().st_size >= min_filesize * 1024 * 1024)):
return True
elif entry.is_dir() and is_recursive:
# 递归搜索子目录
if _search_files(Path(entry.path), is_recursive):
return True
except (OSError, PermissionError):
continue
except (OSError, PermissionError):
pass
return False
pattern = r".*(" + "|".join(extensions) + ")$"
# 遍历目录及子目录
for matched_glob in glob('**', root_dir=directory, recursive=recursive, include_hidden=True):
path = directory.joinpath(matched_glob)
if path.is_file() \
and re.match(pattern, path.name, re.IGNORECASE) \
and path.stat().st_size >= min_filesize * 1024 * 1024:
return True
return False
return _search_files(directory, recursive)
@staticmethod
def list_sub_files(directory: Path, extensions: list) -> List[Path]:
@@ -292,12 +314,20 @@ class SystemUtils:
return [directory]
files = []
pattern = r".*(" + "|".join(extensions) + ")$"
# 遍历目录
for path in directory.iterdir():
if path.is_file() and re.match(pattern, path.name, re.IGNORECASE):
files.append(path)
# 预编译正则表达式
if extensions:
pattern = re.compile(r".*(" + "|".join(extensions) + r")$", re.IGNORECASE)
else:
pattern = re.compile(r".*")
try:
with os.scandir(directory) as entries:
for entry in entries:
if entry.is_file() and pattern.match(entry.name):
files.append(Path(entry.path))
except OSError:
pass
return files
@@ -346,7 +376,7 @@ class SystemUtils:
return items
@staticmethod
def get_directory_size(path: Path) -> float:
def get_directory_size(path: Path) -> int:
"""
计算目录的大小
@@ -358,14 +388,21 @@ class SystemUtils:
"""
if not path or not path.exists():
return 0
if path.is_file():
return path.stat().st_size
total_size = 0
for path in path.glob('**/*'):
if path.is_file():
total_size += path.stat().st_size
return total_size
def _calc_dir_size(dir_path):
total = 0
try:
with os.scandir(dir_path) as entries:
for entry in entries:
if entry.is_file():
total += entry.stat().st_size
elif entry.is_dir():
total += _calc_dir_size(entry.path)
except OSError:
pass
return total
return _calc_dir_size(path) if path.is_dir() else path.stat().st_size
@staticmethod
def space_usage(dir_list: Union[Path, List[Path]]) -> Tuple[float, float]: