mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-11-11 16:21:09 +08:00
1107 lines
48 KiB
Python
1107 lines
48 KiB
Python
"""整理请求候选构建与任务工作流编排。"""
|
||
|
||
import re
|
||
import traceback
|
||
import uuid
|
||
from copy import deepcopy
|
||
from pathlib import Path
|
||
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
||
|
||
from app.application.configuration import get_configured_system_config
|
||
from app.application.directory import DirectoryHelper
|
||
from app.application.formatting import FormatParser
|
||
from app.application.history import (
|
||
describe_history_gate,
|
||
evaluate_history_gate,
|
||
is_skip_action,
|
||
)
|
||
from app.application.transfer.workflow import (
|
||
TransferAdmission,
|
||
TransferTask,
|
||
)
|
||
from app.chain.media import MediaChain
|
||
from app.chain.storage import StorageChain
|
||
from app.chain.transfer.contract import _TransferOwnerBase
|
||
from app.domain.context import MediaInfo, MusicInfo, TorrentInfo
|
||
from app.domain.meta.metabase import MetaBase
|
||
from app.domain.meta.metamusic import MetaMusic
|
||
from app.runtime.log import logger
|
||
from app.runtime.progress import ProgressHelper
|
||
from app.runtime.stop import runtime_stop_state
|
||
from app.schemas.exception import OperationInterrupted
|
||
from app.schemas.media import resolve_media_identity
|
||
from app.schemas.system import TransferDirectoryConf
|
||
from app.schemas.transfer import EpisodeFormat, TransferInfo
|
||
from app.schemas.types import (
|
||
MediaSource,
|
||
MediaType,
|
||
ProgressKey,
|
||
SystemConfigKey,
|
||
)
|
||
from app.schemas.workflow import FileItem
|
||
|
||
from .request import _TransferCandidatePlanner
|
||
|
||
|
||
class TransferWorkflowOwner(_TransferOwnerBase):
|
||
"""协调请求级候选构建并委托规划、执行与结算 owner。"""
|
||
|
||
@staticmethod
|
||
def _build_transfer_fileitem(torrent: TorrentInfo) -> FileItem:
|
||
"""把下载器任务路径转换为整理链使用的本地文件项。"""
|
||
file_path = torrent.path
|
||
return FileItem(
|
||
storage="local",
|
||
path=file_path.as_posix() + ("/" if file_path.is_dir() else ""),
|
||
type="dir" if not file_path.is_file() else "file",
|
||
name=file_path.name,
|
||
size=file_path.stat().st_size,
|
||
extension=file_path.suffix.lstrip("."),
|
||
)
|
||
|
||
def _TransferChain__get_trans_fileitems(
|
||
self,
|
||
fileitem: FileItem,
|
||
predicate: Optional[Callable[[FileItem, bool], bool]],
|
||
verify_file_exists: bool = True,
|
||
) -> List[Tuple[FileItem, bool]]:
|
||
"""
|
||
获取待整理文件项列表
|
||
|
||
:param fileitem: 源文件项
|
||
:param predicate: 用于筛选目录或文件项
|
||
该函数接收两个参数:
|
||
|
||
- `file_item`: 需要判断的文件项(类型为 `FileItem`)
|
||
- `is_bluray_dir`: 表示该项是否为蓝光原盘目录(布尔值)
|
||
|
||
函数应返回 `True` 表示保留该项,`False` 表示过滤掉
|
||
|
||
若 `predicate` 为 `None`,则默认保留所有项
|
||
:param verify_file_exists: 验证目录或文件是否存在,默认值为 `True`
|
||
"""
|
||
if runtime_stop_state.is_system_stopped:
|
||
raise OperationInterrupted()
|
||
|
||
storagechain = StorageChain()
|
||
|
||
def __is_bluray_sub(_path: str) -> bool:
|
||
"""
|
||
判断是否蓝光原盘目录内的子目录或文件
|
||
"""
|
||
return (
|
||
True if re.search(r"BDMV[/\\]STREAM", _path, re.IGNORECASE) else False
|
||
)
|
||
|
||
def __get_bluray_dir(_storage: str, _path: Path) -> Optional[FileItem]:
|
||
"""
|
||
获取蓝光原盘BDMV目录的上级目录
|
||
"""
|
||
for p in _path.parents:
|
||
if p.name == "BDMV":
|
||
return storagechain.get_file_item(storage=_storage, path=p.parent)
|
||
return None
|
||
|
||
def _apply_predicate(
|
||
file_item: FileItem, is_bluray_dir: bool
|
||
) -> List[Tuple[FileItem, bool]]:
|
||
if predicate is None or predicate(file_item, is_bluray_dir):
|
||
return [(file_item, is_bluray_dir)]
|
||
return []
|
||
|
||
if verify_file_exists:
|
||
latest_fileitem = storagechain.get_item(fileitem)
|
||
if not latest_fileitem:
|
||
logger.warn(f"目录或文件不存在:{fileitem.path}")
|
||
return []
|
||
# 确保从历史记录重新整理时 能获得最新的源文件大小、修改日期等
|
||
fileitem = latest_fileitem
|
||
|
||
# 是否蓝光原盘子目录或文件
|
||
if __is_bluray_sub(fileitem.path):
|
||
if bluray_dir := __get_bluray_dir(fileitem.storage, Path(fileitem.path)):
|
||
# 返回该文件所在的原盘根目录
|
||
return _apply_predicate(bluray_dir, True)
|
||
|
||
# 单文件
|
||
if fileitem.type == "file":
|
||
return _apply_predicate(fileitem, False)
|
||
|
||
# 是否蓝光原盘根目录
|
||
sub_items = storagechain.list_files(fileitem, recursion=False) or []
|
||
if storagechain.contains_bluray_subdirectories(sub_items):
|
||
# 当前目录是原盘根目录,不需要递归
|
||
return _apply_predicate(fileitem, True)
|
||
|
||
# 不是原盘根目录 递归获取目录内需要整理的文件项列表
|
||
return [
|
||
item
|
||
for sub_item in sub_items
|
||
for item in (
|
||
self._TransferChain__get_trans_fileitems(
|
||
sub_item, predicate, verify_file_exists=False
|
||
)
|
||
if sub_item.type == "dir"
|
||
else _apply_predicate(sub_item, False)
|
||
)
|
||
]
|
||
|
||
@staticmethod
|
||
def _get_shared_download_roots(file_path: Path) -> set[str]:
|
||
"""
|
||
获取当前文件所在的共享下载根目录边界。
|
||
|
||
父目录兜底回查只应在种子自身目录内进行,不能越过共享下载根目录,
|
||
否则历史中的单文件/无子目录任务会污染同级其它文件的识别结果。
|
||
"""
|
||
shared_roots: set[str] = set()
|
||
media_type_dirs = {mtype.value for mtype in MediaType}
|
||
media_categories = None
|
||
|
||
for dir_info in DirectoryHelper().get_download_dirs():
|
||
if not dir_info.download_path:
|
||
continue
|
||
|
||
download_root = Path(dir_info.download_path)
|
||
if not file_path.is_relative_to(download_root):
|
||
continue
|
||
|
||
shared_roots.add(download_root.as_posix())
|
||
relative_parts = file_path.relative_to(download_root).parts
|
||
current_root = download_root
|
||
part_index = 0
|
||
media_type = dir_info.media_type
|
||
|
||
if (
|
||
not dir_info.media_type
|
||
and dir_info.download_type_folder
|
||
and len(relative_parts) > part_index
|
||
and relative_parts[part_index] in media_type_dirs
|
||
):
|
||
current_root = current_root / relative_parts[part_index]
|
||
shared_roots.add(current_root.as_posix())
|
||
media_type = relative_parts[part_index]
|
||
part_index += 1
|
||
|
||
if (
|
||
not dir_info.media_category
|
||
and dir_info.download_category_folder
|
||
and len(relative_parts) > part_index
|
||
):
|
||
category_root = current_root / relative_parts[part_index]
|
||
shared_roots.add(category_root.as_posix())
|
||
if media_categories is None:
|
||
media_categories = MediaChain().media_category() or {}
|
||
if media_type:
|
||
category_names = media_categories.get(media_type, [])
|
||
else:
|
||
category_names = {
|
||
category
|
||
for categories in media_categories.values()
|
||
for category in categories
|
||
}
|
||
category_paths = sorted(
|
||
(Path(category).parts for category in category_names if category),
|
||
key=len,
|
||
)
|
||
for category_parts in category_paths:
|
||
relative_category_parts = tuple(
|
||
relative_parts[part_index:part_index + len(category_parts)]
|
||
)
|
||
if relative_category_parts != category_parts:
|
||
continue
|
||
category_root = current_root
|
||
for category_part in category_parts:
|
||
category_root = category_root / category_part
|
||
shared_roots.add(category_root.as_posix())
|
||
|
||
return shared_roots
|
||
|
||
@staticmethod
|
||
def _normalize_transfer_identity(
|
||
mediainfo: Optional[Union[MediaInfo, MusicInfo]],
|
||
mtype: Optional[MediaType],
|
||
media_source: Optional[MediaSource],
|
||
media_id: Optional[str],
|
||
meta: Optional[MetaBase],
|
||
) -> Tuple[
|
||
Optional[Union[MediaInfo, MusicInfo]],
|
||
Optional[MediaSource],
|
||
Optional[str],
|
||
Optional[str],
|
||
]:
|
||
"""
|
||
规范整理请求的媒体身份,并在显式身份缺失时短路。
|
||
|
||
:return: ``(媒体信息、媒体来源、媒体 ID、错误信息)``;错误信息为空表示可继续执行
|
||
"""
|
||
explicit_identity = media_source is not None or media_id is not None
|
||
normalized_source, normalized_media_id = resolve_media_identity(
|
||
media_source=media_source,
|
||
media_id=media_id,
|
||
)
|
||
if explicit_identity and (
|
||
not normalized_source or not normalized_media_id
|
||
):
|
||
return (
|
||
mediainfo,
|
||
normalized_source,
|
||
normalized_media_id,
|
||
"整理任务需要同时提供有效的 media_source 和 media_id",
|
||
)
|
||
if not explicit_identity and mediainfo:
|
||
normalized_source, normalized_media_id = resolve_media_identity(
|
||
media=mediainfo
|
||
)
|
||
if explicit_identity and not mediainfo:
|
||
mediainfo = MediaChain().recognize_media(
|
||
mtype=mtype,
|
||
media_source=normalized_source,
|
||
media_id=normalized_media_id,
|
||
music_type=getattr(meta, "music_type", None),
|
||
)
|
||
if not mediainfo:
|
||
return (
|
||
mediainfo,
|
||
normalized_source,
|
||
normalized_media_id,
|
||
"未识别到媒体信息,"
|
||
f"media_source:{normalized_source},media_id:{normalized_media_id}",
|
||
)
|
||
return mediainfo, normalized_source, normalized_media_id, None
|
||
|
||
def _collect_transfer_candidates(
|
||
self,
|
||
fileitem: FileItem,
|
||
batch_mtype: Optional[MediaType],
|
||
min_filesize: int,
|
||
epformat: Optional[EpisodeFormat],
|
||
season: Optional[int],
|
||
continue_callback: Optional[Callable],
|
||
) -> Tuple[List[Tuple[FileItem, bool]], bool]:
|
||
"""
|
||
收集并过滤本次整理的候选文件。
|
||
|
||
候选遍历只负责发现文件,业务过滤集中在此阶段;返回模板命中状态供公开整理流程
|
||
保持“未命中自定义集数模板时跳过”的旧行为。
|
||
"""
|
||
format_handler = (
|
||
FormatParser(
|
||
eformat=epformat.format,
|
||
details=epformat.detail,
|
||
part=epformat.part,
|
||
offset=epformat.offset,
|
||
)
|
||
if epformat
|
||
else None
|
||
)
|
||
has_template = bool(epformat and epformat.format)
|
||
exclude_words = get_configured_system_config().get(
|
||
SystemConfigKey.TransferExcludeWords
|
||
)
|
||
matched_template = False
|
||
|
||
def keep_candidate(item: FileItem, _is_bluray_dir: bool) -> bool:
|
||
"""候选遍历阶段只响应取消请求,不提前应用业务过滤。"""
|
||
if continue_callback and not continue_callback():
|
||
raise OperationInterrupted()
|
||
return True
|
||
|
||
def is_allowed(item: FileItem, is_bluray_dir: bool) -> bool:
|
||
"""判断候选文件是否符合格式、后缀、大小和屏蔽词约束。"""
|
||
nonlocal matched_template
|
||
if continue_callback and not continue_callback():
|
||
raise OperationInterrupted()
|
||
if has_template and format_handler:
|
||
if not format_handler.match(item.name):
|
||
return False
|
||
matched_template = True
|
||
if batch_mtype == MediaType.MUSIC:
|
||
if self._is_music_lyrics_file(item):
|
||
return not self._is_blocked_by_exclude_words(item.path, exclude_words)
|
||
if not self._is_media_file(item, batch_mtype):
|
||
return False
|
||
if not self._is_allow_filesize(item, min_filesize):
|
||
return False
|
||
elif (
|
||
not is_bluray_dir
|
||
and not self._is_subtitle_file(item)
|
||
and not self._is_audio_file(item)
|
||
):
|
||
if not self._is_media_file(item, batch_mtype):
|
||
return False
|
||
if not self._is_allow_filesize(item, min_filesize):
|
||
return False
|
||
if any(
|
||
marker in item.path
|
||
for marker in ("/@Recycle/", "/#recycle/", "/.", "/@eaDir")
|
||
):
|
||
logger.debug(f"{item.path} 是回收站或隐藏的文件")
|
||
return False
|
||
return not self._is_blocked_by_exclude_words(item.path, exclude_words)
|
||
|
||
candidates = self._TransferChain__get_trans_fileitems(fileitem, predicate=keep_candidate)
|
||
return [
|
||
(item, is_bluray_dir)
|
||
for item, is_bluray_dir in candidates
|
||
if is_allowed(item, is_bluray_dir)
|
||
], matched_template
|
||
|
||
def do_transfer(
|
||
self,
|
||
fileitem: FileItem,
|
||
meta: MetaBase = None,
|
||
mediainfo: Optional[Union[MediaInfo, MusicInfo]] = None,
|
||
mtype: Optional[MediaType] = None,
|
||
media_source: Optional[MediaSource] = None,
|
||
media_id: Optional[str] = None,
|
||
target_directory: TransferDirectoryConf = None,
|
||
target_storage: Optional[str] = None,
|
||
target_path: Path = None,
|
||
transfer_type: Optional[str] = None,
|
||
scrape: Optional[bool] = None,
|
||
library_type_folder: Optional[bool] = None,
|
||
library_category_folder: Optional[bool] = None,
|
||
season: Optional[int] = None,
|
||
epformat: EpisodeFormat = None,
|
||
min_filesize: Optional[int] = 0,
|
||
downloader: Optional[str] = None,
|
||
download_hash: Optional[str] = None,
|
||
force: Optional[bool] = False,
|
||
background: Optional[bool] = True,
|
||
manual: Optional[bool] = False,
|
||
preview: Optional[bool] = False,
|
||
sync_extra_files: Optional[bool] = False,
|
||
cleanup_dest_fileitem: Optional[FileItem] = None,
|
||
continue_callback: Callable = None,
|
||
reorganize: Optional[bool] = False,
|
||
) -> Tuple[bool, Union[str, dict]]:
|
||
"""
|
||
兼容公开整理入口,委托给内部批次执行阶段。
|
||
|
||
公开签名是 API、工作流、监控器和插件共同使用的稳定契约;具体整理阶段保留在
|
||
内部方法中,后续可以独立拆分规划、执行和结算,而不迫使调用方迁移参数。
|
||
"""
|
||
return self._run_transfer_workflow(
|
||
fileitem=fileitem,
|
||
meta=meta,
|
||
mediainfo=mediainfo,
|
||
mtype=mtype,
|
||
media_source=media_source,
|
||
media_id=media_id,
|
||
target_directory=target_directory,
|
||
target_storage=target_storage,
|
||
target_path=target_path,
|
||
transfer_type=transfer_type,
|
||
scrape=scrape,
|
||
library_type_folder=library_type_folder,
|
||
library_category_folder=library_category_folder,
|
||
season=season,
|
||
epformat=epformat,
|
||
min_filesize=min_filesize,
|
||
downloader=downloader,
|
||
download_hash=download_hash,
|
||
force=force,
|
||
background=background,
|
||
manual=manual,
|
||
preview=preview,
|
||
sync_extra_files=sync_extra_files,
|
||
cleanup_dest_fileitem=cleanup_dest_fileitem,
|
||
continue_callback=continue_callback,
|
||
reorganize=reorganize,
|
||
)
|
||
|
||
def _execute_transfer(self, *args: Any, **kwargs: Any) -> Tuple[bool, Union[str, dict]]:
|
||
"""兼容旧内部钩子,统一委托请求工作流 owner。"""
|
||
return self._run_transfer_workflow(*args, **kwargs)
|
||
|
||
def _run_transfer_workflow(
|
||
self,
|
||
fileitem: FileItem,
|
||
meta: MetaBase = None,
|
||
mediainfo: Optional[Union[MediaInfo, MusicInfo]] = None,
|
||
mtype: Optional[MediaType] = None,
|
||
media_source: Optional[MediaSource] = None,
|
||
media_id: Optional[str] = None,
|
||
target_directory: TransferDirectoryConf = None,
|
||
target_storage: Optional[str] = None,
|
||
target_path: Path = None,
|
||
transfer_type: Optional[str] = None,
|
||
scrape: Optional[bool] = None,
|
||
library_type_folder: Optional[bool] = None,
|
||
library_category_folder: Optional[bool] = None,
|
||
season: Optional[int] = None,
|
||
epformat: EpisodeFormat = None,
|
||
min_filesize: Optional[int] = 0,
|
||
downloader: Optional[str] = None,
|
||
download_hash: Optional[str] = None,
|
||
force: Optional[bool] = False,
|
||
background: Optional[bool] = True,
|
||
manual: Optional[bool] = False,
|
||
preview: Optional[bool] = False,
|
||
sync_extra_files: Optional[bool] = False,
|
||
cleanup_dest_fileitem: Optional[FileItem] = None,
|
||
continue_callback: Callable = None,
|
||
reorganize: Optional[bool] = False,
|
||
recovery_admission: Optional[TransferAdmission] = None,
|
||
) -> Tuple[bool, Union[str, dict]]:
|
||
"""
|
||
执行一个复杂目录的整理操作
|
||
:param fileitem: 文件项
|
||
:param meta: 元数据
|
||
:param mediainfo: 媒体信息
|
||
:param mtype: 未提供媒体信息时使用的媒体类型提示
|
||
:param media_source: 请求级识别与刮削数据源
|
||
:param media_id: 数据源原生 ID;显式指定身份时与 media_source 成对传入
|
||
:param target_directory: 目标目录配置
|
||
:param target_storage: 目标存储器
|
||
:param target_path: 目标路径
|
||
:param transfer_type: 整理类型
|
||
:param scrape: 是否刮削元数据
|
||
:param library_type_folder: 媒体库类型子目录
|
||
:param library_category_folder: 媒体库类别子目录
|
||
:param season: 季
|
||
:param epformat: 剧集格式
|
||
:param min_filesize: 最小文件大小(MB)
|
||
:param downloader: 下载器
|
||
:param download_hash: 下载记录hash
|
||
:param force: 是否强制整理
|
||
:param background: 是否后台运行
|
||
:param manual: 是否手动整理
|
||
:param preview: 是否仅预览
|
||
:param reorganize: 是否清理已有成功记录后重新整理
|
||
:param sync_extra_files: 是否在整理主视频文件时同步整理同媒体附加文件
|
||
:param cleanup_dest_fileitem: 确认存在待整理任务后需要清理的旧目标文件
|
||
:param continue_callback: 继续处理回调
|
||
:param recovery_admission: 内部恢复调用绑定的既有 durable 记录
|
||
返回:成功标识,错误信息
|
||
"""
|
||
mediainfo, media_source, media_id, identity_error = (
|
||
self._normalize_transfer_identity(
|
||
mediainfo=mediainfo,
|
||
mtype=mtype,
|
||
media_source=media_source,
|
||
media_id=media_id,
|
||
meta=meta,
|
||
)
|
||
)
|
||
if identity_error:
|
||
return False, identity_error
|
||
|
||
# 是否全部成功
|
||
all_success = True
|
||
transfer_batch_id = str(uuid.uuid4())
|
||
batch_mtype = getattr(mediainfo, "type", None)
|
||
if batch_mtype in (None, MediaType.UNKNOWN):
|
||
batch_mtype = mtype
|
||
if preview:
|
||
# 预览模式始终同步执行,避免进入异步队列
|
||
background = False
|
||
# 自定义格式
|
||
has_episode_format_template = bool(epformat and epformat.format)
|
||
formaterHandler = (
|
||
FormatParser(
|
||
eformat=epformat.format,
|
||
details=epformat.detail,
|
||
part=epformat.part,
|
||
offset=epformat.offset,
|
||
)
|
||
if epformat
|
||
else None
|
||
)
|
||
|
||
# 汇总错误信息
|
||
err_msgs: List[str] = []
|
||
transfer_exclude_words = get_configured_system_config().get(
|
||
SystemConfigKey.TransferExcludeWords
|
||
)
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
candidate_planner = _TransferCandidatePlanner(
|
||
self,
|
||
meta=meta,
|
||
season=season,
|
||
formater=formaterHandler,
|
||
batch_mtype=batch_mtype,
|
||
mediainfo=mediainfo,
|
||
continue_callback=continue_callback,
|
||
has_episode_format_template=has_episode_format_template,
|
||
transfer_exclude_words=transfer_exclude_words,
|
||
download_hash=download_hash,
|
||
sync_extra_files=bool(sync_extra_files),
|
||
fileitem=fileitem,
|
||
)
|
||
|
||
try:
|
||
file_items, matched_episode_format_template = self._collect_transfer_candidates(
|
||
fileitem=fileitem,
|
||
batch_mtype=batch_mtype,
|
||
min_filesize=min_filesize or 0,
|
||
epformat=epformat,
|
||
season=season,
|
||
continue_callback=continue_callback,
|
||
)
|
||
except OperationInterrupted:
|
||
return False, f"{fileitem.name} 已取消"
|
||
|
||
if not file_items:
|
||
if has_episode_format_template and not matched_episode_format_template:
|
||
logger.info(f"{fileitem.path} 未匹配到集数定位模板,跳过整理")
|
||
if preview:
|
||
return True, {
|
||
"summary": {"total": 0, "success": 0, "failed": 0},
|
||
"items": [],
|
||
"message": "",
|
||
}
|
||
return True, ""
|
||
logger.warn(f"{fileitem.path} 没有找到可整理的媒体文件")
|
||
return False, f"{fileitem.name} 没有找到可整理的媒体文件"
|
||
|
||
file_items, inherited_meta_map = candidate_planner._plan_file_items(file_items)
|
||
|
||
planned_file_count = len(file_items)
|
||
|
||
if preview:
|
||
logger.info(f"正在预览 {planned_file_count} 个文件的整理路径...")
|
||
else:
|
||
logger.info(f"正在计划整理 {planned_file_count} 个文件...")
|
||
|
||
try:
|
||
(
|
||
transfer_tasks,
|
||
all_success,
|
||
err_msgs,
|
||
skipped_history_count,
|
||
skipped_torrents,
|
||
) = self._build_transfer_tasks(
|
||
file_items=file_items,
|
||
inherited_meta_map=inherited_meta_map,
|
||
build_file_meta=candidate_planner._build_file_meta,
|
||
meta=meta,
|
||
mediainfo=mediainfo,
|
||
media_source=media_source,
|
||
media_id=media_id,
|
||
batch_mtype=batch_mtype,
|
||
target_directory=target_directory,
|
||
target_storage=target_storage,
|
||
target_path=target_path,
|
||
transfer_type=transfer_type,
|
||
scrape=scrape,
|
||
library_type_folder=library_type_folder,
|
||
library_category_folder=library_category_folder,
|
||
downloader=downloader,
|
||
download_hash=download_hash,
|
||
transfer_batch_id=transfer_batch_id,
|
||
manual=bool(manual),
|
||
background=bool(background),
|
||
preview=bool(preview),
|
||
reorganize=bool(reorganize),
|
||
force=bool(force),
|
||
continue_callback=continue_callback,
|
||
cleanup_dest_fileitem=cleanup_dest_fileitem,
|
||
recovery_admission=recovery_admission,
|
||
)
|
||
except OperationInterrupted:
|
||
return False, f"{fileitem.name} 已取消"
|
||
|
||
all_success, err_msgs, preview_items = self._execute_transfer_tasks(
|
||
transfer_tasks=transfer_tasks,
|
||
preview=bool(preview),
|
||
continue_callback=continue_callback,
|
||
all_success=all_success,
|
||
err_msgs=err_msgs,
|
||
)
|
||
|
||
# 下载器任务在这一轮可能因为历史记录全部命中而没有进入整理队列,
|
||
# 这里补打一遍已整理标签,避免同一种子被重复扫描。
|
||
if (
|
||
skipped_history_count == planned_file_count
|
||
and skipped_torrents
|
||
):
|
||
for skipped_hash, skipped_downloader in skipped_torrents:
|
||
logger.info(f"补充设置下载任务已整理标签:{skipped_hash}")
|
||
self._TransferChain__mark_torrent_completed_if_done(
|
||
skipped_hash, skipped_downloader
|
||
)
|
||
|
||
error_msg = "、".join(err_msgs[:2]) + (
|
||
f",等{len(err_msgs)}个文件错误!" if len(err_msgs) > 2 else ""
|
||
)
|
||
if preview:
|
||
return all_success, {
|
||
"summary": {
|
||
"total": len(preview_items),
|
||
"success": len([item for item in preview_items if item.get("success")]),
|
||
"failed": len([item for item in preview_items if not item.get("success")]),
|
||
},
|
||
"items": preview_items,
|
||
"message": error_msg,
|
||
}
|
||
return all_success, error_msg
|
||
|
||
def _build_transfer_tasks(
|
||
self,
|
||
*,
|
||
file_items: List[Tuple[FileItem, bool]],
|
||
inherited_meta_map: Dict[Tuple[str, str], MetaBase],
|
||
build_file_meta: Callable[[Path, Optional[List[str]]], Optional[MetaBase]],
|
||
meta: Optional[MetaBase],
|
||
mediainfo: Optional[Union[MediaInfo, MusicInfo]],
|
||
media_source: Optional[MediaSource],
|
||
media_id: Optional[str],
|
||
batch_mtype: Optional[MediaType],
|
||
target_directory: Optional[TransferDirectoryConf],
|
||
target_storage: Optional[str],
|
||
target_path: Optional[Path],
|
||
transfer_type: Optional[str],
|
||
scrape: Optional[bool],
|
||
library_type_folder: Optional[bool],
|
||
library_category_folder: Optional[bool],
|
||
downloader: Optional[str],
|
||
download_hash: Optional[str],
|
||
transfer_batch_id: str,
|
||
manual: bool,
|
||
background: bool,
|
||
preview: bool,
|
||
reorganize: bool,
|
||
force: bool,
|
||
continue_callback: Optional[Callable[[], bool]],
|
||
cleanup_dest_fileitem: Optional[FileItem],
|
||
recovery_admission: Optional[TransferAdmission],
|
||
) -> Tuple[List[TransferTask], bool, List[str], int, set[Tuple[str, str]]]:
|
||
"""从冻结候选构建任务,并集中执行历史去重与 durable 绑定。"""
|
||
_build_file_meta = build_file_meta
|
||
# 整理所有文件
|
||
all_success = True
|
||
transfer_tasks: List[TransferTask] = []
|
||
err_msgs: List[str] = []
|
||
skipped_history_count = 0
|
||
skipped_torrents = set()
|
||
cleanup_intent_assigned = False
|
||
try:
|
||
for file_item, bluray_dir in file_items:
|
||
if runtime_stop_state.is_system_stopped:
|
||
raise OperationInterrupted()
|
||
if continue_callback and not continue_callback():
|
||
raise OperationInterrupted()
|
||
file_path = Path(file_item.path)
|
||
|
||
# 自动整理按 app/application/history.py 的统一判定去重(失败记录放行重试、
|
||
# 成功但源文件已变化放行交 overwrite_mode 决断);手动整理可清理失败记录,
|
||
# 或按用户确认清理成功记录;手动显式指定媒体身份时,先解除旧失败任务再重新规划。
|
||
if (not force or reorganize or (manual and (media_source is not None or media_id is not None))) and not preview:
|
||
transfer_history_oper = self.transfer_history_repository
|
||
transferd = self._get_manual_transfer_history(
|
||
fileitem=file_item,
|
||
transfer_history_oper=transfer_history_oper,
|
||
include_move_dest=bool(manual and reorganize),
|
||
)
|
||
if transferd:
|
||
should_reorganize = manual and (
|
||
reorganize or not transferd.status
|
||
)
|
||
if should_reorganize:
|
||
if not reorganize and not (media_source is not None or media_id is not None):
|
||
durable_retry = self._request_durable_transfer_retry(
|
||
transferd, requested_by="manual_reorganize",
|
||
)
|
||
if durable_retry is not None:
|
||
accepted, message = durable_retry
|
||
if accepted:
|
||
logger.info(message)
|
||
else:
|
||
all_success = False
|
||
logger.error(message)
|
||
err_msgs.append(message)
|
||
# 普通失败重试仍由唯一 durable 调度器继续原计划。
|
||
continue
|
||
state, message = self._delete_manual_transfer_history(
|
||
history=transferd,
|
||
transfer_history_oper=transfer_history_oper,
|
||
)
|
||
if not state:
|
||
all_success = False
|
||
logger.error(message)
|
||
err_msgs.append(message)
|
||
continue
|
||
logger.info(
|
||
f"{file_item.path} 已清理旧整理记录,继续重新整理。"
|
||
)
|
||
transferd = None
|
||
|
||
if transferd:
|
||
history_description = describe_history_gate(
|
||
transferd,
|
||
file_size=file_item.size,
|
||
file_modify_time=file_item.modify_time,
|
||
fileid=file_item.fileid,
|
||
)
|
||
if not manual:
|
||
# 自动路径(目录监控、下载器轮询)与监控分发共用同一套判定,
|
||
# 否则监控层刚放行的失败重试与升级请求会在这里被全额收回
|
||
gate_action = evaluate_history_gate(
|
||
transferd,
|
||
file_size=file_item.size,
|
||
file_modify_time=file_item.modify_time,
|
||
fileid=file_item.fileid,
|
||
)
|
||
if not is_skip_action(gate_action):
|
||
logger.info(
|
||
f"{file_item.path} 命中"
|
||
f"{history_description}"
|
||
f",重新送入整理"
|
||
)
|
||
transferd = None
|
||
|
||
if transferd:
|
||
skipped_history_count += 1
|
||
if not transferd.status:
|
||
all_success = False
|
||
# 失败记录能走到这里说明重试次数已用尽,此时同样要打已整理标签让种子
|
||
# 退出轮询,否则下载器每一轮都会重新扫描并在这里被拦一次,空转且刷屏
|
||
candidate_hash = download_hash or transferd.download_hash
|
||
candidate_downloader = downloader or transferd.downloader
|
||
if candidate_hash and candidate_downloader:
|
||
skipped_torrents.add(
|
||
(candidate_hash, candidate_downloader)
|
||
)
|
||
logger.info(
|
||
f"{file_item.path} 已整理过("
|
||
f"{history_description}"
|
||
f"),如需重新处理,请删除整理记录。"
|
||
)
|
||
err_msgs.append(f"{file_item.name} 已整理过")
|
||
continue
|
||
|
||
# 提前获取下载历史,以便获取自定义识别词
|
||
download_history_repository = self.download_history_repository
|
||
download_history = self._resolve_download_history(
|
||
repository=download_history_repository,
|
||
file_path=file_path,
|
||
bluray_dir=bluray_dir,
|
||
download_hash=download_hash,
|
||
)
|
||
|
||
history_music_meta, history_music_info = self._restore_music_download_context(
|
||
download_history=download_history,
|
||
file_path=file_path,
|
||
)
|
||
|
||
if not meta:
|
||
# 文件元数据(优先使用订阅识别词)
|
||
inherited_meta = inherited_meta_map.get(
|
||
self._get_file_key(file_item)
|
||
)
|
||
if history_music_meta:
|
||
file_meta = history_music_meta
|
||
elif inherited_meta:
|
||
file_meta = deepcopy(inherited_meta)
|
||
else:
|
||
file_meta = _build_file_meta(
|
||
file_path,
|
||
self._get_subscribe_custom_words(download_history),
|
||
)
|
||
else:
|
||
file_meta = _build_file_meta(file_path, None)
|
||
|
||
if not file_meta:
|
||
all_success = False
|
||
logger.error(f"{file_path.name} 无法识别有效信息")
|
||
err_msgs.append(f"{file_path.name} 无法识别有效信息")
|
||
continue
|
||
|
||
# 获取下载Hash
|
||
if download_history and (not downloader or not download_hash):
|
||
_downloader = download_history.downloader
|
||
_download_hash = download_history.download_hash
|
||
else:
|
||
_downloader = downloader
|
||
_download_hash = download_hash
|
||
|
||
# 自动整理预载的媒体信息来自整条下载历史;电影合集内文件年份冲突时逐文件识别。
|
||
task_mediainfo = mediainfo or history_music_info
|
||
if not task_mediainfo and isinstance(file_meta, MetaMusic):
|
||
# 无标签音频按目录级专辑匹配补齐曲目身份,命中结果带缓存不会逐文件重复请求
|
||
file_meta, task_mediainfo = self._match_music_album_context(
|
||
file_item, file_path, file_meta
|
||
)
|
||
if (
|
||
not manual
|
||
and task_mediainfo
|
||
and self._is_movie_year_conflict(file_meta, task_mediainfo)
|
||
):
|
||
task_mediainfo = None
|
||
|
||
# 后台整理
|
||
transfer_task = TransferTask(
|
||
fileitem=file_item,
|
||
meta=file_meta,
|
||
mediainfo=task_mediainfo,
|
||
media_source=media_source,
|
||
media_id=media_id,
|
||
mtype=batch_mtype,
|
||
target_directory=target_directory,
|
||
target_storage=target_storage,
|
||
target_path=target_path,
|
||
transfer_type=transfer_type,
|
||
scrape=scrape,
|
||
library_type_folder=library_type_folder,
|
||
library_category_folder=library_category_folder,
|
||
downloader=_downloader,
|
||
download_hash=_download_hash,
|
||
download_history=download_history,
|
||
transfer_batch_id=transfer_batch_id,
|
||
manual=manual,
|
||
background=background,
|
||
preview=preview,
|
||
)
|
||
cleanup_intent = (
|
||
cleanup_dest_fileitem
|
||
if not preview and not cleanup_intent_assigned
|
||
else None
|
||
)
|
||
transfer_task.bind_planning_input(
|
||
self._TransferChain__build_planning_input(
|
||
transfer_task,
|
||
cleanup_dest_fileitem=cleanup_intent,
|
||
)
|
||
)
|
||
if (
|
||
recovery_admission
|
||
and file_item.storage == recovery_admission.storage
|
||
and file_item.path == recovery_admission.src_path
|
||
):
|
||
transfer_task.bind_admission_task_id(recovery_admission.task_id)
|
||
self._TransferChain__bind_claimed_admission(
|
||
transfer_task,
|
||
recovery_admission,
|
||
)
|
||
if recovery_admission.planning_input:
|
||
transfer_task.bind_planning_input(
|
||
recovery_admission.planning_input
|
||
)
|
||
if recovery_admission.checkpoint:
|
||
transfer_task.bind_plan_checkpoint(
|
||
recovery_admission.checkpoint
|
||
)
|
||
if background:
|
||
try:
|
||
queued = self.put_to_queue(task=transfer_task)
|
||
except Exception as err:
|
||
all_success = False
|
||
message = f"{file_path.name} 加入整理队列失败:{err}"
|
||
err_msgs.append(message)
|
||
logger.error(message)
|
||
continue
|
||
if queued:
|
||
if cleanup_intent:
|
||
cleanup_intent_assigned = True
|
||
logger.info(f"{file_path.name} 已添加到整理队列")
|
||
else:
|
||
logger.debug(f"{file_path.name} 已在整理队列中,跳过")
|
||
else:
|
||
# 加入列表
|
||
if self._TransferChain__put_to_jobview(transfer_task):
|
||
self._register_scrape_batch_task(transfer_task)
|
||
transfer_tasks.append(transfer_task)
|
||
if cleanup_intent:
|
||
cleanup_intent_assigned = True
|
||
else:
|
||
logger.debug(f"{file_path.name} 已在整理列表中,跳过")
|
||
except OperationInterrupted:
|
||
raise
|
||
finally:
|
||
file_items.clear()
|
||
del file_items
|
||
self._close_scrape_batch(transfer_batch_id)
|
||
|
||
return (
|
||
transfer_tasks,
|
||
all_success,
|
||
err_msgs,
|
||
skipped_history_count,
|
||
skipped_torrents,
|
||
)
|
||
|
||
def _execute_transfer_tasks(
|
||
self,
|
||
*,
|
||
transfer_tasks: List[TransferTask],
|
||
preview: bool,
|
||
continue_callback: Optional[Callable[[], bool]],
|
||
all_success: bool,
|
||
err_msgs: List[str],
|
||
) -> Tuple[bool, List[str], List[dict[str, Any]]]:
|
||
"""同步消费已规划任务;后台任务只由 queue owner 消费。"""
|
||
# 实时整理
|
||
preview_items: List[dict[str, Any]] = []
|
||
|
||
def _preview_callback(task: TransferTask, transferinfo: TransferInfo) -> Tuple[bool, str]:
|
||
item_meta = task.meta
|
||
item_media = task.mediainfo
|
||
preview_items.append(
|
||
{
|
||
"source": task.fileitem.path,
|
||
"target": transferinfo.target_item.path if transferinfo.target_item else None,
|
||
"target_dir": transferinfo.target_diritem.path if transferinfo.target_diritem else None,
|
||
"success": transferinfo.success,
|
||
"message": transferinfo.message,
|
||
"type": item_media.type.value if item_media and item_media.type else None,
|
||
"title": item_media.title_year if item_media else None,
|
||
"season": item_meta.begin_season if item_meta else None,
|
||
"episode": item_meta.begin_episode if item_meta else None,
|
||
"episode_end": item_meta.end_episode if item_meta else None,
|
||
"part": item_meta.part if item_meta else None,
|
||
"org_string": item_meta.org_string if item_meta else None,
|
||
"apply_words": item_meta.apply_words if item_meta else [],
|
||
"resource_team": item_meta.resource_team if item_meta else None,
|
||
"customization": item_meta.customization if item_meta else None,
|
||
}
|
||
)
|
||
return transferinfo.success, transferinfo.message
|
||
|
||
if transfer_tasks:
|
||
# 总数量
|
||
total_num = len(transfer_tasks)
|
||
# 已处理数量
|
||
processed_num = 0
|
||
# 失败数量
|
||
fail_num = 0
|
||
# 已完成文件
|
||
finished_files = []
|
||
|
||
progress = None
|
||
if not preview:
|
||
# 启动进度
|
||
progress = ProgressHelper(ProgressKey.FileTransfer)
|
||
progress.start()
|
||
__process_msg = f"开始整理,共 {total_num} 个文件 ..."
|
||
logger.info(__process_msg)
|
||
progress.update(value=0, text=__process_msg)
|
||
try:
|
||
for transfer_task in transfer_tasks:
|
||
if runtime_stop_state.is_system_stopped:
|
||
break
|
||
if continue_callback and not continue_callback():
|
||
break
|
||
if not preview:
|
||
# 更新进度
|
||
__process_msg = f"正在整理 ({processed_num + fail_num + 1}/{total_num}){transfer_task.fileitem.name} ..."
|
||
logger.info(__process_msg)
|
||
progress.update(
|
||
value=(processed_num + fail_num) / total_num * 100,
|
||
text=__process_msg,
|
||
data={
|
||
"current": Path(transfer_task.fileitem.path).as_posix(),
|
||
"finished": finished_files,
|
||
},
|
||
)
|
||
terminal = False
|
||
terminal_settlement: Optional[bool] = None
|
||
|
||
def callback_after_terminal_settlement(
|
||
callback_task: TransferTask,
|
||
transferinfo: TransferInfo,
|
||
) -> Tuple[bool, str]:
|
||
"""同步路径也由默认回调原子提交历史、事件与 durable 终态。"""
|
||
nonlocal terminal_settlement
|
||
callback = (
|
||
_preview_callback
|
||
if preview
|
||
else self._TransferChain__default_callback
|
||
)
|
||
try:
|
||
return callback(callback_task, transferinfo)
|
||
finally:
|
||
if not callback_task.preview:
|
||
terminal_settlement = callback_task.terminal_settled
|
||
|
||
try:
|
||
self._TransferChain__claim_task_for_execution(transfer_task)
|
||
self._TransferChain__start_job_execution(transfer_task)
|
||
state, err_msg = self._TransferChain__handle_transfer(
|
||
task=transfer_task,
|
||
callback=callback_after_terminal_settlement,
|
||
)
|
||
terminal = bool(preview or transfer_task.plan_checkpoint is not None)
|
||
except Exception as e:
|
||
if terminal_settlement is not None:
|
||
terminal = True
|
||
logger.error(
|
||
f"{transfer_task.fileitem.name} 整理任务处理出现错误:"
|
||
f"{e} - {traceback.format_exc()}"
|
||
)
|
||
if not preview:
|
||
self._TransferChain__fail_transfer_task(transfer_task)
|
||
state, err_msg = False, str(e)
|
||
finally:
|
||
durable_settled = self._TransferChain__finish_job_execution(
|
||
transfer_task,
|
||
terminal=terminal,
|
||
terminal_settlement=terminal_settlement,
|
||
)
|
||
if terminal and not durable_settled:
|
||
state = False
|
||
err_msg = "整理任务 durable 终态结算失去租约"
|
||
if not state:
|
||
all_success = False
|
||
logger.warn(f"{transfer_task.fileitem.name} {err_msg}")
|
||
err_msgs.append(f"{transfer_task.fileitem.name} {err_msg}")
|
||
if preview:
|
||
# 预览模式不走默认回调,这里需要手动收敛任务状态,避免残留 running
|
||
self.jobview.fail_task(transfer_task)
|
||
self.jobview.try_remove_job(transfer_task)
|
||
if preview and (
|
||
not preview_items or preview_items[-1].get("source") != transfer_task.fileitem.path):
|
||
preview_items.append(
|
||
{
|
||
"source": transfer_task.fileitem.path,
|
||
"target": None,
|
||
"target_dir": None,
|
||
"success": False,
|
||
"message": err_msg,
|
||
"type": None,
|
||
"title": None,
|
||
"season": transfer_task.meta.begin_season if transfer_task.meta else None,
|
||
"episode": transfer_task.meta.begin_episode if transfer_task.meta else None,
|
||
"episode_end": transfer_task.meta.end_episode if transfer_task.meta else None,
|
||
"part": transfer_task.meta.part if transfer_task.meta else None,
|
||
"org_string": transfer_task.meta.org_string if transfer_task.meta else None,
|
||
"apply_words": transfer_task.meta.apply_words if transfer_task.meta else [],
|
||
"resource_team": transfer_task.meta.resource_team if transfer_task.meta else None,
|
||
"customization": transfer_task.meta.customization if transfer_task.meta else None,
|
||
}
|
||
)
|
||
fail_num += 1
|
||
else:
|
||
if preview:
|
||
# 预览模式手动标记完成,确保可重复预览
|
||
self.jobview.finish_task(transfer_task)
|
||
self.jobview.try_remove_job(transfer_task)
|
||
processed_num += 1
|
||
# 记录已完成
|
||
finished_files.append(Path(transfer_task.fileitem.path).as_posix())
|
||
finally:
|
||
transfer_tasks.clear()
|
||
del transfer_tasks
|
||
|
||
# 整理结束
|
||
if not preview:
|
||
__end_msg = (
|
||
f"整理队列处理完成,共整理 {total_num} 个文件,失败 {fail_num} 个"
|
||
)
|
||
logger.info(__end_msg)
|
||
progress.update(value=100, text=__end_msg, data={})
|
||
progress.end()
|
||
|
||
return all_success, err_msgs, preview_items
|