mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-02-10 05:56:37 +08:00
fix(transfer): 修复部分情况下无法正确统计已完成任务总大小的问题
- get_directory_size 使用 os.scandir 递归遍历提升性能 - 当任务文件项存储类型为 local 时,若其大小为空,则通过 SystemUtils 获取目录大小以确保 完成任务的准确统计。 fix(cache): 修改 fresh 和 async_fresh 默认参数为 True refactor(filemanager): 移除整理后总大小计算逻辑 - 删除 TransHandler 中对整理目录总大小的冗余计算,提升性能并简化流程。 perf(system): 使用 scandir 优化文件扫描性能 - 重构 SystemUtils 中的文件扫描方法(list_files、exists_file、list_sub_files), - 采用 os.scandir 替代 glob 实现,并预编译正则表达式以提升目录遍历与文件匹配性能。
This commit is contained in:
@@ -33,6 +33,7 @@ from app.schemas.types import TorrentStatus, EventType, MediaType, ProgressKey,
|
||||
SystemConfigKey, ChainEventType, ContentType
|
||||
from app.utils.singleton import Singleton
|
||||
from app.utils.string import StringUtils
|
||||
from app.utils.system import SystemUtils
|
||||
|
||||
downloader_lock = threading.Lock()
|
||||
job_lock = threading.Lock()
|
||||
@@ -329,8 +330,12 @@ class JobManager:
|
||||
# 计算状态为完成的任务数
|
||||
if __mediaid__ not in self._job_view:
|
||||
return 0
|
||||
return sum([task.fileitem.size for task in self._job_view[__mediaid__].tasks if
|
||||
task.state == "completed" and task.fileitem.size is not None])
|
||||
return sum([
|
||||
task.fileitem.size if task.fileitem.size is not None
|
||||
else (SystemUtils.get_directory_size(Path(task.fileitem.path)) if task.fileitem.storage == "local" else 0)
|
||||
for task in self._job_view[__mediaid__].tasks
|
||||
if task.state == "completed"
|
||||
])
|
||||
|
||||
def total(self) -> int:
|
||||
"""
|
||||
|
||||
@@ -1016,7 +1016,7 @@ class AsyncFileBackend(AsyncCacheBackend):
|
||||
|
||||
|
||||
@contextmanager
|
||||
def fresh(fresh: bool = False):
|
||||
def fresh(fresh: bool = True):
|
||||
"""
|
||||
是否获取新数据(不使用缓存的值)
|
||||
|
||||
@@ -1031,7 +1031,7 @@ def fresh(fresh: bool = False):
|
||||
_fresh.reset(token)
|
||||
|
||||
@asynccontextmanager
|
||||
async def async_fresh(fresh: bool = False):
|
||||
async def async_fresh(fresh: bool = True):
|
||||
"""
|
||||
是否获取新数据(不使用缓存的值)
|
||||
|
||||
|
||||
@@ -150,14 +150,11 @@ class TransHandler:
|
||||
return self.result.copy()
|
||||
|
||||
logger.info(f"文件夹 {fileitem.path} 整理成功")
|
||||
# 计算目录下所有文件大小
|
||||
total_size = sum(file.stat().st_size for file in Path(fileitem.path).rglob('*') if file.is_file())
|
||||
# 返回整理后的路径
|
||||
self.__set_result(success=True,
|
||||
fileitem=fileitem,
|
||||
target_item=new_diritem,
|
||||
target_diritem=new_diritem,
|
||||
total_size=total_size,
|
||||
need_scrape=need_scrape,
|
||||
need_notify=need_notify,
|
||||
transfer_type=transfer_type)
|
||||
|
||||
@@ -7,7 +7,6 @@ import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import uuid
|
||||
from glob import glob
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Tuple, Union
|
||||
|
||||
@@ -225,23 +224,31 @@ class SystemUtils:
|
||||
if directory.is_file():
|
||||
return [directory]
|
||||
|
||||
if not min_filesize:
|
||||
min_filesize = 0
|
||||
|
||||
files = []
|
||||
# 预编译正则表达式
|
||||
if extensions:
|
||||
pattern = r".*(" + "|".join(extensions) + ")$"
|
||||
pattern = re.compile(r".*(" + "|".join(extensions) + r")$", re.IGNORECASE)
|
||||
else:
|
||||
pattern = r".*"
|
||||
pattern = re.compile(r".*")
|
||||
|
||||
# 遍历目录及子目录
|
||||
for matched_glob in glob('**', root_dir=directory, recursive=recursive, include_hidden=True):
|
||||
path = directory.joinpath(matched_glob)
|
||||
if path.is_file() \
|
||||
and re.match(pattern, path.name, re.IGNORECASE) \
|
||||
and path.stat().st_size >= min_filesize * 1024 * 1024:
|
||||
files.append(path)
|
||||
def _scan_directory(dir_path: Path, is_recursive: bool):
|
||||
try:
|
||||
with os.scandir(dir_path) as entries:
|
||||
for entry in entries:
|
||||
try:
|
||||
if entry.is_file(follow_symlinks=False):
|
||||
entry_path = Path(entry.path)
|
||||
if (pattern.match(entry.name) and
|
||||
(min_filesize <= 0 or entry.stat().st_size >= min_filesize * 1024 * 1024)):
|
||||
files.append(entry_path)
|
||||
elif entry.is_dir() and is_recursive:
|
||||
_scan_directory(Path(entry.path), is_recursive)
|
||||
except (OSError, PermissionError):
|
||||
continue
|
||||
except (OSError, PermissionError):
|
||||
pass
|
||||
|
||||
_scan_directory(directory, recursive)
|
||||
return files
|
||||
|
||||
@staticmethod
|
||||
@@ -256,29 +263,44 @@ class SystemUtils:
|
||||
:return: True存在 False不存在
|
||||
"""
|
||||
|
||||
if not min_filesize:
|
||||
min_filesize = 0
|
||||
|
||||
if not directory.exists():
|
||||
return False
|
||||
|
||||
# 预编译正则表达式
|
||||
if extensions:
|
||||
pattern = re.compile(r".*(" + "|".join(extensions) + r")$", re.IGNORECASE)
|
||||
else:
|
||||
pattern = re.compile(r".*")
|
||||
|
||||
if directory.is_file():
|
||||
# 检查单个文件是否符合条件
|
||||
if extensions and not pattern.match(directory.name):
|
||||
return False
|
||||
if min_filesize > 0 and directory.stat().st_size < min_filesize * 1024 * 1024:
|
||||
return False
|
||||
return True
|
||||
|
||||
if not min_filesize:
|
||||
min_filesize = 0
|
||||
def _search_files(dir_path: Path, is_recursive: bool) -> bool:
|
||||
try:
|
||||
with os.scandir(dir_path) as entries:
|
||||
for entry in entries:
|
||||
try:
|
||||
if entry.is_file(follow_symlinks=False):
|
||||
# 检查文件是否符合条件
|
||||
if (pattern.match(entry.name) and
|
||||
(min_filesize <= 0 or entry.stat().st_size >= min_filesize * 1024 * 1024)):
|
||||
return True
|
||||
elif entry.is_dir() and is_recursive:
|
||||
# 递归搜索子目录
|
||||
if _search_files(Path(entry.path), is_recursive):
|
||||
return True
|
||||
except (OSError, PermissionError):
|
||||
continue
|
||||
except (OSError, PermissionError):
|
||||
pass
|
||||
return False
|
||||
|
||||
pattern = r".*(" + "|".join(extensions) + ")$"
|
||||
|
||||
# 遍历目录及子目录
|
||||
for matched_glob in glob('**', root_dir=directory, recursive=recursive, include_hidden=True):
|
||||
path = directory.joinpath(matched_glob)
|
||||
if path.is_file() \
|
||||
and re.match(pattern, path.name, re.IGNORECASE) \
|
||||
and path.stat().st_size >= min_filesize * 1024 * 1024:
|
||||
return True
|
||||
|
||||
return False
|
||||
return _search_files(directory, recursive)
|
||||
|
||||
@staticmethod
|
||||
def list_sub_files(directory: Path, extensions: list) -> List[Path]:
|
||||
@@ -292,12 +314,20 @@ class SystemUtils:
|
||||
return [directory]
|
||||
|
||||
files = []
|
||||
pattern = r".*(" + "|".join(extensions) + ")$"
|
||||
|
||||
# 遍历目录
|
||||
for path in directory.iterdir():
|
||||
if path.is_file() and re.match(pattern, path.name, re.IGNORECASE):
|
||||
files.append(path)
|
||||
# 预编译正则表达式
|
||||
if extensions:
|
||||
pattern = re.compile(r".*(" + "|".join(extensions) + r")$", re.IGNORECASE)
|
||||
else:
|
||||
pattern = re.compile(r".*")
|
||||
|
||||
try:
|
||||
with os.scandir(directory) as entries:
|
||||
for entry in entries:
|
||||
if entry.is_file() and pattern.match(entry.name):
|
||||
files.append(Path(entry.path))
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
return files
|
||||
|
||||
@@ -346,7 +376,7 @@ class SystemUtils:
|
||||
return items
|
||||
|
||||
@staticmethod
|
||||
def get_directory_size(path: Path) -> float:
|
||||
def get_directory_size(path: Path) -> int:
|
||||
"""
|
||||
计算目录的大小
|
||||
|
||||
@@ -358,14 +388,21 @@ class SystemUtils:
|
||||
"""
|
||||
if not path or not path.exists():
|
||||
return 0
|
||||
if path.is_file():
|
||||
return path.stat().st_size
|
||||
total_size = 0
|
||||
for path in path.glob('**/*'):
|
||||
if path.is_file():
|
||||
total_size += path.stat().st_size
|
||||
|
||||
return total_size
|
||||
def _calc_dir_size(dir_path):
|
||||
total = 0
|
||||
try:
|
||||
with os.scandir(dir_path) as entries:
|
||||
for entry in entries:
|
||||
if entry.is_file():
|
||||
total += entry.stat().st_size
|
||||
elif entry.is_dir():
|
||||
total += _calc_dir_size(entry.path)
|
||||
except OSError:
|
||||
pass
|
||||
return total
|
||||
|
||||
return _calc_dir_size(path) if path.is_dir() else path.stat().st_size
|
||||
|
||||
@staticmethod
|
||||
def space_usage(dir_list: Union[Path, List[Path]]) -> Tuple[float, float]:
|
||||
|
||||
Reference in New Issue
Block a user