From 0ee21b38decc8849523be5eb209a6275284f7469 Mon Sep 17 00:00:00 2001 From: Attente <19653207+wikrin@users.noreply.github.com> Date: Fri, 25 Oct 2024 01:27:39 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=20fix:=20-=20=E4=BF=AE=E5=A4=8D=E5=9B=A0?= =?UTF-8?q?=E9=A6=96=E4=B8=AA=E5=AD=90=E7=9B=AE=E5=BD=95=E4=B8=AD=E6=97=A0?= =?UTF-8?q?=E7=9B=AE=E6=A0=87=E6=96=87=E4=BB=B6=E8=80=8C=E4=B8=8D=E5=A4=84?= =?UTF-8?q?=E7=90=86=E6=95=B4=E4=B8=AA=E6=96=87=E4=BB=B6=E5=A4=B9=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98=20-=20=E6=B7=BB=E5=8A=A0=E5=90=8C=E6=97=B6?= =?UTF-8?q?=E6=95=B4=E7=90=86=E9=9F=B3=E8=BD=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/chain/transfer.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/app/chain/transfer.py b/app/chain/transfer.py index 0b5472fa..d294015c 100644 --- a/app/chain/transfer.py +++ b/app/chain/transfer.py @@ -184,6 +184,17 @@ class TransferChain(ChainBase): # 开始进度 self.progress.start(ProgressKey.FileTransfer) + # 汇总季集清单 + season_episodes: Dict[Tuple, List[int]] = {} + # 汇总元数据 + metas: Dict[Tuple, MetaBase] = {} + # 汇总媒体信息 + medias: Dict[Tuple, MediaInfo] = {} + # 汇总整理信息 + transfers: Dict[Tuple, TransferInfo] = {} + + # 待整理文件列表 + file_items = [] # 汇总错误信息 err_msgs: List[str] = [] # 已处理数量 @@ -209,14 +220,6 @@ class TransferChain(ChainBase): # 处理所有待整理目录或文件，默认一个整理路径或文件只有一个媒体信息 for trans_item in trans_items: - # 汇总季集清单 - season_episodes: Dict[Tuple, List[int]] = {} - # 汇总元数据 - metas: Dict[Tuple, MetaBase] = {} - # 汇总媒体信息 - medias: Dict[Tuple, MediaInfo] = {} - # 汇总整理信息 - transfers: Dict[Tuple, TransferInfo] = {} item_path = Path(trans_item.path) # 如果是目录且不是⼀蓝光原盘，获取所有文件并整理 @@ -228,7 +231,7 @@ class TransferChain(ChainBase): continue else: # 文件或蓝光目录 - file_items = [trans_item] + file_items.append(trans_item) if formaterHandler: # 有集自定义格式，过滤文件 @@ -236,13 +239,15 @@ class TransferChain(ChainBase): # 过滤后缀和大小 file_items = [f for f in file_items - if f.extension and (f".{f.extension.lower()}" in self.all_exts + if f.extension and (f".{f.extension.lower()}" in self.all_exts + settings.RMT_AUDIO_TRACK_EXT and (not min_filesize or f.size > min_filesize * 1024 * 1024))] if not file_items: logger.warn(f"{fileitem.path} 没有找到可整理的媒体文件") return False, f"{fileitem.name} 没有找到可整理的媒体文件" + # 更新总文件数 + total_num = len(file_items) logger.info(f"正在整理 {len(file_items)} 个文件...") # 整理所有文件 From d108b0da78301c103c9e2f1200a61e4fd2f5840d Mon Sep 17 00:00:00 2001 From: Attente <19653207+wikrin@users.noreply.github.com> Date: Fri, 25 Oct 2024 01:28:59 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E4=BB=85=E5=8F=96=E6=B6=88=E7=BC=A9?= =?UTF-8?q?=E8=BF=9B,=E6=B2=A1=E6=9C=89=E5=85=B6=E4=BB=96=E4=BB=BB?= =?UTF-8?q?=E4=BD=95=E6=94=B9=E5=8A=A8=20=E5=87=8F=E5=B0=91`for`=E5=B5=8C?= =?UTF-8?q?=E5=A5=97,=20=E6=B1=87=E6=80=BB=E9=81=8D=E5=8E=86=E7=9B=AE?= =?UTF-8?q?=E5=BD=95,=20=E8=BF=99=E6=A0=B7=E8=83=BD=E6=8F=90=E4=BE=9B?= =?UTF-8?q?=E6=9B=B4=E5=87=86=E7=A1=AE=E7=9A=84`=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E6=95=B0`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/chain/transfer.py | 442 +++++++++++++++++++++--------------------- 1 file changed, 221 insertions(+), 221 deletions(-) diff --git a/app/chain/transfer.py b/app/chain/transfer.py index d294015c..e71318e6 100644 --- a/app/chain/transfer.py +++ b/app/chain/transfer.py @@ -233,256 +233,256 @@ class TransferChain(ChainBase): # 文件或蓝光目录 file_items.append(trans_item) - if formaterHandler: - # 有集自定义格式，过滤文件 - file_items = [f for f in file_items if formaterHandler.match(f.name)] + if formaterHandler: + # 有集自定义格式，过滤文件 + file_items = [f for f in file_items if formaterHandler.match(f.name)] - # 过滤后缀和大小 - file_items = [f for f in file_items - if f.extension and (f".{f.extension.lower()}" in self.all_exts + settings.RMT_AUDIO_TRACK_EXT - and (not min_filesize or f.size > min_filesize * 1024 * 1024))] + # 过滤后缀和大小 + file_items = [f for f in file_items + if f.extension and (f".{f.extension.lower()}" in self.all_exts + settings.RMT_AUDIO_TRACK_EXT + and (not min_filesize or f.size > min_filesize * 1024 * 1024))] - if not file_items: - logger.warn(f"{fileitem.path} 没有找到可整理的媒体文件") - return False, f"{fileitem.name} 没有找到可整理的媒体文件" + if not file_items: + logger.warn(f"{fileitem.path} 没有找到可整理的媒体文件") + return False, f"{fileitem.name} 没有找到可整理的媒体文件" - # 更新总文件数 - total_num = len(file_items) - logger.info(f"正在整理 {len(file_items)} 个文件...") + # 更新总文件数 + total_num = len(file_items) + logger.info(f"正在整理 {len(file_items)} 个文件...") - # 整理所有文件 - for file_item in file_items: - if global_vars.is_system_stopped: - break - file_path = Path(file_item.path) - # 回收站及隐藏的文件不处理 - if file_item.path.find('/@Recycle/') != -1 \ - or file_item.path.find('/#recycle/') != -1 \ - or file_item.path.find('/.') != -1 \ - or file_item.path.find('/@eaDir') != -1: - logger.debug(f"{file_item.path} 是回收站或隐藏的文件") - # 计数 - processed_num += 1 - skip_num += 1 - continue + # 整理所有文件 + for file_item in file_items: + if global_vars.is_system_stopped: + break + file_path = Path(file_item.path) + # 回收站及隐藏的文件不处理 + if file_item.path.find('/@Recycle/') != -1 \ + or file_item.path.find('/#recycle/') != -1 \ + or file_item.path.find('/.') != -1 \ + or file_item.path.find('/@eaDir') != -1: + logger.debug(f"{file_item.path} 是回收站或隐藏的文件") + # 计数 + processed_num += 1 + skip_num += 1 + continue - # 整理屏蔽词不处理 - is_blocked = False - if transfer_exclude_words: - for keyword in transfer_exclude_words: - if not keyword: - continue - if keyword and re.search(r"%s" % keyword, file_item.path, re.IGNORECASE): - logger.info(f"{file_item.path} 命中整理屏蔽词 {keyword}，不处理") - is_blocked = True - break - if is_blocked: - err_msgs.append(f"{file_item.name} 命中整理屏蔽词") - # 计数 - processed_num += 1 - skip_num += 1 - continue - - # 整理成功的不再处理 - if not force: - transferd = self.transferhis.get_by_src(file_item.path, storage=file_item.storage) - if transferd and transferd.status: - logger.info(f"{file_item.path} 已成功整理过，如需重新处理，请删除历史记录。") - # 计数 - processed_num += 1 - skip_num += 1 + # 整理屏蔽词不处理 + is_blocked = False + if transfer_exclude_words: + for keyword in transfer_exclude_words: + if not keyword: continue + if keyword and re.search(r"%s" % keyword, file_item.path, re.IGNORECASE): + logger.info(f"{file_item.path} 命中整理屏蔽词 {keyword}，不处理") + is_blocked = True + break + if is_blocked: + err_msgs.append(f"{file_item.name} 命中整理屏蔽词") + # 计数 + processed_num += 1 + skip_num += 1 + continue - # 更新进度 - self.progress.update(value=processed_num / total_num * 100, - text=f"正在整理 （{processed_num + 1}/{total_num}）{file_item.name} ...", - key=ProgressKey.FileTransfer) - - if not meta: - # 文件元数据 - file_meta = MetaInfoPath(file_path) - else: - file_meta = meta - - # 合并季 - if season is not None: - file_meta.begin_season = season - - if not file_meta: - logger.error(f"{file_path} 无法识别有效信息") - err_msgs.append(f"{file_path} 无法识别有效信息") + # 整理成功的不再处理 + if not force: + transferd = self.transferhis.get_by_src(file_item.path, storage=file_item.storage) + if transferd and transferd.status: + logger.info(f"{file_item.path} 已成功整理过，如需重新处理，请删除历史记录。") # 计数 processed_num += 1 - fail_num += 1 + skip_num += 1 continue - # 自定义识别 - if formaterHandler: - # 开始集、结束集、PART - begin_ep, end_ep, part = formaterHandler.split_episode(file_path.name) - if begin_ep is not None: - file_meta.begin_episode = begin_ep - file_meta.part = part - if end_ep is not None: - file_meta.end_episode = end_ep + # 更新进度 + self.progress.update(value=processed_num / total_num * 100, + text=f"正在整理 （{processed_num + 1}/{total_num}）{file_item.name} ...", + key=ProgressKey.FileTransfer) - if not mediainfo: - # 识别媒体信息 - file_mediainfo = self.mediachain.recognize_by_meta(file_meta) - else: - file_mediainfo = mediainfo + if not meta: + # 文件元数据 + file_meta = MetaInfoPath(file_path) + else: + file_meta = meta - if not file_mediainfo: - logger.warn(f'{file_path} 未识别到媒体信息') - # 新增整理失败历史记录 - his = self.transferhis.add_fail( - fileitem=file_item, - mode=transfer_type, - meta=file_meta, - download_hash=download_hash - ) - self.post_message(Notification( - mtype=NotificationType.Manual, - title=f"{file_path.name} 未识别到媒体信息，无法入库！", - text=f"回复：```\n/redo {his.id} [tmdbid]|[类型]\n``` 手动识别整理。", - link=settings.MP_DOMAIN('#/history') - )) - # 计数 - processed_num += 1 - fail_num += 1 - continue + # 合并季 + if season is not None: + file_meta.begin_season = season - # 如果未开启新增已入库媒体是否跟随TMDB信息变化则根据tmdbid查询之前的title - if not settings.SCRAP_FOLLOW_TMDB: - transfer_history = self.transferhis.get_by_type_tmdbid(tmdbid=file_mediainfo.tmdb_id, - mtype=file_mediainfo.type.value) - if transfer_history: - file_mediainfo.title = transfer_history.title + if not file_meta: + logger.error(f"{file_path} 无法识别有效信息") + err_msgs.append(f"{file_path} 无法识别有效信息") + # 计数 + processed_num += 1 + fail_num += 1 + continue - logger.info(f"{file_path.name} 识别为：{file_mediainfo.type.value} {file_mediainfo.title_year}") + # 自定义识别 + if formaterHandler: + # 开始集、结束集、PART + begin_ep, end_ep, part = formaterHandler.split_episode(file_path.name) + if begin_ep is not None: + file_meta.begin_episode = begin_ep + file_meta.part = part + if end_ep is not None: + file_meta.end_episode = end_ep - # 获取集数据 - if file_mediainfo.type == MediaType.TV: - if file_meta.begin_season is None: - file_meta.begin_season = 1 - file_mediainfo.season = file_mediainfo.season or file_meta.begin_season - episodes_info = self.tmdbchain.tmdb_episodes( - tmdbid=file_mediainfo.tmdb_id, - season=file_mediainfo.season - ) - else: - episodes_info = None + if not mediainfo: + # 识别媒体信息 + file_mediainfo = self.mediachain.recognize_by_meta(file_meta) + else: + file_mediainfo = mediainfo - # 获取下载hash - if not download_hash: - download_file = self.downloadhis.get_file_by_fullpath(file_item.path) - if download_file: - download_hash = download_file.download_hash - - # 执行整理 - transferinfo: TransferInfo = self.transfer(fileitem=file_item, - meta=file_meta, - mediainfo=file_mediainfo, - transfer_type=transfer_type, - target_storage=target_storage, - target_path=target_path, - episodes_info=episodes_info, - scrape=scrape) - if not transferinfo: - logger.error("文件整理模块运行失败") - return False, "文件整理模块运行失败" - if not transferinfo.success: - # 整理失败 - logger.warn(f"{file_path.name} 入库失败：{transferinfo.message}") - err_msgs.append(f"{file_path.name} {transferinfo.message}") - # 新增整理失败历史记录 - self.transferhis.add_fail( - fileitem=file_item, - mode=transfer_type, - download_hash=download_hash, - meta=file_meta, - mediainfo=file_mediainfo, - transferinfo=transferinfo - ) - # 发送消息 - self.post_message(Notification( - mtype=NotificationType.Manual, - title=f"{file_mediainfo.title_year} {file_meta.season_episode} 入库失败！", - text=f"原因：{transferinfo.message or '未知'}", - image=file_mediainfo.get_message_image(), - link=settings.MP_DOMAIN('#/history') - )) - # 计数 - processed_num += 1 - fail_num += 1 - continue - - # 汇总信息 - current_transfer_type = transferinfo.transfer_type - mkey = (file_mediainfo.tmdb_id, file_meta.begin_season) - if mkey not in medias: - # 新增信息 - metas[mkey] = file_meta - medias[mkey] = file_mediainfo - season_episodes[mkey] = file_meta.episode_list - transfers[mkey] = transferinfo - else: - # 合并季集清单 - season_episodes[mkey] = list(set(season_episodes[mkey] + file_meta.episode_list)) - # 合并整理数据 - transfers[mkey].file_count += transferinfo.file_count - transfers[mkey].total_size += transferinfo.total_size - transfers[mkey].file_list.extend(transferinfo.file_list) - transfers[mkey].file_list_new.extend(transferinfo.file_list_new) - transfers[mkey].fail_list.extend(transferinfo.fail_list) - - # 新增整理成功历史记录 - self.transferhis.add_success( + if not file_mediainfo: + logger.warn(f'{file_path} 未识别到媒体信息') + # 新增整理失败历史记录 + his = self.transferhis.add_fail( fileitem=file_item, - mode=transfer_type or transferinfo.transfer_type, + mode=transfer_type, + meta=file_meta, + download_hash=download_hash + ) + self.post_message(Notification( + mtype=NotificationType.Manual, + title=f"{file_path.name} 未识别到媒体信息，无法入库！", + text=f"回复：```\n/redo {his.id} [tmdbid]|[类型]\n``` 手动识别整理。", + link=settings.MP_DOMAIN('#/history') + )) + # 计数 + processed_num += 1 + fail_num += 1 + continue + + # 如果未开启新增已入库媒体是否跟随TMDB信息变化则根据tmdbid查询之前的title + if not settings.SCRAP_FOLLOW_TMDB: + transfer_history = self.transferhis.get_by_type_tmdbid(tmdbid=file_mediainfo.tmdb_id, + mtype=file_mediainfo.type.value) + if transfer_history: + file_mediainfo.title = transfer_history.title + + logger.info(f"{file_path.name} 识别为：{file_mediainfo.type.value} {file_mediainfo.title_year}") + + # 获取集数据 + if file_mediainfo.type == MediaType.TV: + if file_meta.begin_season is None: + file_meta.begin_season = 1 + file_mediainfo.season = file_mediainfo.season or file_meta.begin_season + episodes_info = self.tmdbchain.tmdb_episodes( + tmdbid=file_mediainfo.tmdb_id, + season=file_mediainfo.season + ) + else: + episodes_info = None + + # 获取下载hash + if not download_hash: + download_file = self.downloadhis.get_file_by_fullpath(file_item.path) + if download_file: + download_hash = download_file.download_hash + + # 执行整理 + transferinfo: TransferInfo = self.transfer(fileitem=file_item, + meta=file_meta, + mediainfo=file_mediainfo, + transfer_type=transfer_type, + target_storage=target_storage, + target_path=target_path, + episodes_info=episodes_info, + scrape=scrape) + if not transferinfo: + logger.error("文件整理模块运行失败") + return False, "文件整理模块运行失败" + if not transferinfo.success: + # 整理失败 + logger.warn(f"{file_path.name} 入库失败：{transferinfo.message}") + err_msgs.append(f"{file_path.name} {transferinfo.message}") + # 新增整理失败历史记录 + self.transferhis.add_fail( + fileitem=file_item, + mode=transfer_type, download_hash=download_hash, meta=file_meta, mediainfo=file_mediainfo, transferinfo=transferinfo ) - - # 更新进度 + # 发送消息 + self.post_message(Notification( + mtype=NotificationType.Manual, + title=f"{file_mediainfo.title_year} {file_meta.season_episode} 入库失败！", + text=f"原因：{transferinfo.message or '未知'}", + image=file_mediainfo.get_message_image(), + link=settings.MP_DOMAIN('#/history') + )) + # 计数 processed_num += 1 - self.progress.update(value=processed_num / total_num * 100, - text=f"{file_path.name} 整理完成", - key=ProgressKey.FileTransfer) + fail_num += 1 + continue - # 目录或文件整理完成 - self.progress.update(text=f"{trans_item.path} 整理完成，正在执行后续处理 ...", - key=ProgressKey.FileTransfer) + # 汇总信息 + current_transfer_type = transferinfo.transfer_type + mkey = (file_mediainfo.tmdb_id, file_meta.begin_season) + if mkey not in medias: + # 新增信息 + metas[mkey] = file_meta + medias[mkey] = file_mediainfo + season_episodes[mkey] = file_meta.episode_list + transfers[mkey] = transferinfo + else: + # 合并季集清单 + season_episodes[mkey] = list(set(season_episodes[mkey] + file_meta.episode_list)) + # 合并整理数据 + transfers[mkey].file_count += transferinfo.file_count + transfers[mkey].total_size += transferinfo.total_size + transfers[mkey].file_list.extend(transferinfo.file_list) + transfers[mkey].file_list_new.extend(transferinfo.file_list_new) + transfers[mkey].fail_list.extend(transferinfo.fail_list) - # 执行后续处理 - for mkey, media in medias.items(): - transfer_meta = metas[mkey] - transfer_info = transfers[mkey] - # 发送通知 - if transfer_info.need_notify: - se_str = None - if media.type == MediaType.TV: - se_str = f"{transfer_meta.season} {StringUtils.format_ep(season_episodes[mkey])}" - self.send_transfer_message(meta=transfer_meta, - mediainfo=media, - transferinfo=transfer_info, - season_episode=se_str) - # 刮削事件 - if scrape or transfer_info.need_scrape: - self.eventmanager.send_event(EventType.MetadataScrape, { - 'meta': transfer_meta, - 'mediainfo': media, - 'fileitem': transfer_info.target_diritem - }) - # 整理完成事件 - self.eventmanager.send_event(EventType.TransferComplete, { + # 新增整理成功历史记录 + self.transferhis.add_success( + fileitem=file_item, + mode=transfer_type or transferinfo.transfer_type, + download_hash=download_hash, + meta=file_meta, + mediainfo=file_mediainfo, + transferinfo=transferinfo + ) + + # 更新进度 + processed_num += 1 + self.progress.update(value=processed_num / total_num * 100, + text=f"{file_path.name} 整理完成", + key=ProgressKey.FileTransfer) + + # 目录或文件整理完成 + self.progress.update(text=f"{trans_item.path} 整理完成，正在执行后续处理 ...", + key=ProgressKey.FileTransfer) + + # 执行后续处理 + for mkey, media in medias.items(): + transfer_meta = metas[mkey] + transfer_info = transfers[mkey] + # 发送通知 + if transfer_info.need_notify: + se_str = None + if media.type == MediaType.TV: + se_str = f"{transfer_meta.season} {StringUtils.format_ep(season_episodes[mkey])}" + self.send_transfer_message(meta=transfer_meta, + mediainfo=media, + transferinfo=transfer_info, + season_episode=se_str) + # 刮削事件 + if scrape or transfer_info.need_scrape: + self.eventmanager.send_event(EventType.MetadataScrape, { 'meta': transfer_meta, 'mediainfo': media, - 'transferinfo': transfer_info, - 'download_hash': download_hash, + 'fileitem': transfer_info.target_diritem }) + # 整理完成事件 + self.eventmanager.send_event(EventType.TransferComplete, { + 'meta': transfer_meta, + 'mediainfo': media, + 'transferinfo': transfer_info, + 'download_hash': download_hash, + }) # 移动模式处理 if current_transfer_type in ["move"]: From 1f6fa22aa15729f2e7dac79159fa77e9090d7b03 Mon Sep 17 00:00:00 2001 From: Attente <19653207+wikrin@users.noreply.github.com> Date: Fri, 25 Oct 2024 02:54:41 +0800 Subject: [PATCH 3/3] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20`storagechain.l?= =?UTF-8?q?ist=5Ffiles`=20=E9=80=92=E5=BD=92=E5=BE=97=E5=88=B0=E7=9A=84?= =?UTF-8?q?=E5=88=97=E8=A1=A8=E8=A2=AB=E8=A6=86=E7=9B=96=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/chain/transfer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/chain/transfer.py b/app/chain/transfer.py index e71318e6..54164329 100644 --- a/app/chain/transfer.py +++ b/app/chain/transfer.py @@ -227,8 +227,8 @@ class TransferChain(ChainBase): and not (trans_item.storage == "local" and SystemUtils.is_bluray_dir(item_path))): # 遍历获取下载目录所有文件（递归） file_items = self.storagechain.list_files(trans_item, recursion=True) - if not file_items: - continue + if file_items: + file_items.extend(trans_items) else: # 文件或蓝光目录 file_items.append(trans_item) @@ -248,7 +248,7 @@ class TransferChain(ChainBase): # 更新总文件数 total_num = len(file_items) - logger.info(f"正在整理 {len(file_items)} 个文件...") + logger.info(f"正在整理 {total_num} 个文件...") # 整理所有文件 for file_item in file_items: