From ee8f7dd1a259e66ab54cdb103894c2d16ee794cd Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Sun, 23 Apr 2023 21:35:57 +0800 Subject: [PATCH] =?UTF-8?q?-=20=E6=96=B0=E5=A2=9E=E8=87=AA=E5=AE=9A?= =?UTF-8?q?=E4=B9=89=E5=9F=9F=E5=90=8D=20-=20=E5=A2=9E=E5=8A=A0=E8=BF=9E?= =?UTF-8?q?=E6=8E=A5=E6=8A=A5=E9=94=99=E5=9F=9F=E5=90=8D=E6=8F=90=E7=A4=BA?= =?UTF-8?q?=20#195=20-=20=E4=BF=AE=E5=A4=8D=20Dev-debug=20=E5=BC=80?= =?UTF-8?q?=E5=90=AF=E7=9A=84=E9=94=99=E8=AF=AF=20#192=20-=20=E9=87=8D?= =?UTF-8?q?=E5=81=9A=E9=87=8D=E5=91=BD=E5=90=8D=E9=80=BB=E8=BE=91=EF=BC=8C?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E5=90=88=E9=9B=86=E9=87=8D=E5=91=BD=E5=90=8D?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E6=96=87=E4=BB=B6=E5=A4=B9=E5=86=85=E9=87=8D?= =?UTF-8?q?=E5=91=BD=E5=90=8D=EF=BC=8C=E6=94=AF=E6=8C=81=E5=AD=97=E5=B9=95?= =?UTF-8?q?=E9=87=8D=E5=91=BD=E5=90=8D=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/module/conf/config.py | 87 ++++++--------- src/module/core/download_client.py | 13 ++- src/module/downloader/qb_downloader.py | 14 ++- src/module/manager/renamer.py | 117 ++++++++++++++++++-- src/module/models/__init__.py | 1 + src/module/models/config.py | 69 ++++++++++++ src/module/network/request_url.py | 4 +- src/module/parser/analyser/rename_parser.py | 60 ++++++---- src/module/parser/title_parser.py | 9 +- 9 files changed, 276 insertions(+), 98 deletions(-) create mode 100644 src/module/models/config.py diff --git a/src/module/conf/config.py b/src/module/conf/config.py index 346034f5..9570512b 100644 --- a/src/module/conf/config.py +++ b/src/module/conf/config.py @@ -2,9 +2,8 @@ import json import os import logging -from dataclasses import dataclass - -from .const import DEFAULT_SETTINGS, ENV_TO_ATTR +from module.conf.const import ENV_TO_ATTR +from module.models import Config logger = logging.getLogger(__name__) @@ -15,70 +14,48 @@ except ImportError: VERSION = "DEV_VERSION" -class ConfLoad(dict): - def __getattr__(self, item): - return self.get(item) - - def __setattr__(self, key, value): - self[key] = value +def save_config_to_file(config: Config, path: str): + with open(path, "w", encoding="utf-8") as f: + json.dump(config, f, indent=4) + logger.info(f"Config saved") -@dataclass -class Settings: - program: ConfLoad - downloader: ConfLoad - rss_parser: ConfLoad - bangumi_manage: ConfLoad - debug: ConfLoad - proxy: ConfLoad - notification: ConfLoad +def load_config_from_file(path: str) -> Config: + with open(path, "r", encoding="utf-8") as f: + config = json.load(f) + return config - def __init__(self, path: str | None): - self.load(path) - def load(self, path: str | None): - if path is None: - conf = DEFAULT_SETTINGS - elif os.path.isfile(path): - with open(path, "r") as f: - # Use utf-8 to avoid encoding error - conf = json.load(f, encoding="utf-8") +def _val_from_env(env: str, attr: tuple): + if isinstance(attr, tuple): + if attr[1] == "bool": + return os.environ[env].lower() == "true" + elif attr[1] == "int": + return int(os.environ[env]) + elif attr[1] == "float": + return float(os.environ[env]) else: - conf = self._create_config() - for key, section in conf.items(): - setattr(self, key, ConfLoad(section)) + return os.environ[env] + else: + return os.environ[env] - @staticmethod - def _val_from_env(env, attr): - val = os.environ[env] - if isinstance(attr, tuple): - conv_func = attr[1] - val = conv_func(val) - return val - def _create_config(self): - _settings = DEFAULT_SETTINGS - for key, section in ENV_TO_ATTR.items(): - for env, attr in section.items(): - if env in os.environ: - attr_name = attr[0] if isinstance(attr, tuple) else attr - _settings[key][attr_name] = self._val_from_env(env, attr) - with open(CONFIG_PATH, "w") as f: - # Save utf-8 to avoid encoding error - json.dump(_settings, f, indent=4, ensure_ascii=False) - logger.warning(f"Config file had been transferred from environment variables to {CONFIG_PATH}, some settings may be lost.") - logger.warning("Please check the config file and restart the program.") - logger.warning("Please check github wiki (https://github.com/EstrellaXD/Auto_Bangumi/#/wiki) for more information.") - return _settings +def env_to_config() -> Config: + _settings = Config() + for key, section in ENV_TO_ATTR.items(): + for env, attr in section.items(): + if env in os.environ: + attr_name = attr[0] if isinstance(attr, tuple) else attr + setattr(_settings, attr_name, _val_from_env(env, attr)) + return _settings if os.path.isdir("config") and VERSION == "DEV_VERSION": CONFIG_PATH = "config/config_dev.json" + settings = load_config_from_file(CONFIG_PATH) + print(dict(settings)) elif os.path.isdir("config") and VERSION != "DEV_VERSION": CONFIG_PATH = "config/config.json" -else: - CONFIG_PATH = None - -settings = Settings(CONFIG_PATH) + diff --git a/src/module/core/download_client.py b/src/module/core/download_client.py index 6d92b47d..2800d028 100644 --- a/src/module/core/download_client.py +++ b/src/module/core/download_client.py @@ -74,16 +74,16 @@ class DownloadClient: # logger.info("to rule.") logger.debug("Finished.") - def get_torrent_info(self): + def get_torrent_info(self, category="Bangumi"): return self.client.torrents_info( - status_filter="completed", category="Bangumi" + status_filter="completed", category=category ) - def rename_torrent_file(self, hash, new_file_name, old_path, new_path): + def rename_torrent_file(self, _hash, old_path, new_path): self.client.torrents_rename_file( - torrent_hash=hash, new_file_name=new_file_name, old_path=old_path, new_path=new_path + torrent_hash=_hash, old_path=old_path, new_path=new_path ) - logger.info(f"{old_path} >> {new_path}, new name {new_file_name}") + logger.info(f"{old_path} >> {new_path}") def delete_torrent(self, hashes): self.client.torrents_delete( @@ -114,3 +114,6 @@ class DownloadClient: def get_torrent_path(self, hashes): return self.client.get_torrent_path(hashes) + def set_category(self, hashes, category): + self.client.set_category(hashes, category) + diff --git a/src/module/downloader/qb_downloader.py b/src/module/downloader/qb_downloader.py index c9fe9dbe..4170dfb5 100644 --- a/src/module/downloader/qb_downloader.py +++ b/src/module/downloader/qb_downloader.py @@ -19,6 +19,8 @@ class QbDownloader: host=host, username=username, password=password, + VERIFY_WEBUI_CERTIFICATE=settings.downloader.ssl, + RAISE_ERROR_FOR_UNSUPPORTED_QBITTORRENT_VERSIONS=True, ) while True: try: @@ -56,9 +58,8 @@ class QbDownloader: torrent_hashes=hash ) - def torrents_rename_file(self, torrent_hash, new_file_name, old_path, new_path): - self._client.torrents_rename_file(torrent_hash=torrent_hash, new_file_name=new_file_name, - old_path=old_path, new_path=new_path) + def torrents_rename_file(self, torrent_hash, old_path, new_path): + self._client.torrents_rename_file(torrent_hash=torrent_hash, old_path=old_path, new_path=new_path) def get_rss_info(self): item = self._client.rss_items().get("Mikan_RSS") @@ -92,5 +93,8 @@ class QbDownloader: def get_download_rule(self): return self._client.rss_rules() - def get_torrent_path(self, hash): - return self._client.torrents_info(hashes=hash)[0].save_path + def get_torrent_path(self, _hash): + return self._client.torrents_info(hashes=_hash)[0].save_path + + def set_category(self, _hash, category): + self._client.torrents_set_category(category, hashes=_hash) diff --git a/src/module/manager/renamer.py b/src/module/manager/renamer.py index 0782377c..e02e3ecb 100644 --- a/src/module/manager/renamer.py +++ b/src/module/manager/renamer.py @@ -7,7 +7,6 @@ from module.core.download_client import DownloadClient from module.conf import settings from module.parser import TitleParser -from module.network import PostNotification, ServerChanNotification logger = logging.getLogger(__name__) @@ -23,8 +22,8 @@ class Renamer: logger.info(f"Finished checking {torrent_count} files' name, renamed {rename_count} files.") logger.debug(f"Checked {torrent_count} files") - def get_torrent_info(self): - recent_info = self.client.get_torrent_info() + def get_torrent_info(self, category="Bangumi"): + recent_info = self.client.get_torrent_info(category=category) torrent_count = len(recent_info) return recent_info, torrent_count @@ -65,9 +64,8 @@ class Renamer: try: new_name = self._renamer.download_parser(name, folder_name, season, suffix, settings.bangumi_manage.rename_method) if path_name != new_name: - old_path = info.content_path.replace(info.save_path, "") - old_path = old_path[len(os.path.sep):] - self.client.rename_torrent_file(torrent_hash, new_name, old_path, new_name) + old_path = info.content_path.replace(info.save_path, "")[len(os.path.sep):] + self.client.rename_torrent_file(torrent_hash, old_path, new_name) rename_count += 1 else: continue @@ -85,6 +83,111 @@ class Renamer: torrent_hash = info.hash _, season, folder_name, _, download_path = self.split_path(info.content_path) new_path = os.path.join(settings.downloader.path, folder_name, f"Season {season}") - # print(new_path) self.client.move_torrent(torrent_hash, new_path) + @staticmethod + def check_files(info, suffix_type: str = "media"): + if suffix_type == "subtitle": + suffix_list = [".ass", ".srt"] + else: + suffix_list = [".mp4", ".mkv"] + file_list = [] + for f in info.files: + file_name = f.name + suffix = os.path.splitext(file_name)[-1] + if suffix in suffix_list: + file_list.append(file_name) + return file_list + + def rename_file(self, info, media_path): + old_name = info.name + suffix = os.path.splitext(media_path)[-1] + compare_name = media_path.split(os.path.sep)[-1] + folder_name, season = self.get_folder_and_season(info.save_path) + new_path = self._renamer.download_parser(old_name, folder_name, season, suffix) + if compare_name != new_path: + try: + self.client.rename_torrent_file(_hash=info.hash, old_path=media_path, new_path=new_path) + except Exception as e: + logger.warning(f"{old_name} rename failed") + logger.warning(f"Folder name: {folder_name}, Season: {season}, Suffix: {suffix}") + logger.debug(e) + + def rename_collection(self, info, media_list: list[str]): + folder_name, season = self.get_folder_and_season(info.save_path) + _hash = info.hash + for media_path in media_list: + path_len = len(media_path.split(os.path.sep)) + if path_len <= 2: + suffix = os.path.splitext(media_path)[-1] + old_name = media_path.split(os.path.sep)[-1] + new_name = self._renamer.download_parser(old_name, folder_name, season, suffix) + if old_name != new_name: + try: + self.client.rename_torrent_file(_hash=_hash, old_path=media_path, new_path=new_name) + except Exception as e: + logger.warning(f"{old_name} rename failed") + logger.warning(f"Folder name: {folder_name}, Season: {season}, Suffix: {suffix}") + logger.debug(e) + self.client.set_category(category="BangumiCollection", hashes=_hash) + + def rename_subtitles(self, subtitle_list: list[str], media_old_name, media_new_name, _hash): + for subtitle_file in subtitle_list: + if re.search(media_old_name, subtitle_file) is not None: + subtitle_lang = subtitle_file.split(".")[-2] + new_subtitle_name = f"{media_new_name}.{subtitle_lang}.ass" + self.client.rename_torrent_file(_hash, subtitle_file, new_subtitle_name) + logger.info(f"Rename subtitles for {media_old_name} to {media_new_name}") + + + @staticmethod + def get_folder_and_season(save_path: str): + # Remove default save path + save_path = save_path.replace(settings.downloader.path, "") + # Check windows or linux path + path_parts = PurePath(save_path).parts \ + if PurePath(save_path).name != save_path \ + else PureWindowsPath(save_path).parts + # Get folder name + folder_name = path_parts[1] if path_parts[0] == "/" else path_parts[0] + # Get season + try: + if re.search(r"S\d{1,2}|[Ss]eason", path_parts[-1]) is not None: + season = int(re.search(r"\d{1,2}", path_parts[-1]).group()) + else: + season = 1 + except Exception as e: + logger.debug(e) + logger.debug("No Season info") + season = 1 + return folder_name, season + + def rename(self): + # Get torrent info + recent_info, torrent_count = self.get_torrent_info() + rename_count = 0 + for info in recent_info: + try: + media_list = self.check_files(info) + if len(media_list) == 1: + self.rename_file(info, media_list[0]) + rename_count += 1 + # TODO: Rename subtitles + elif len(media_list) > 1: + logger.info("Start rename collection") + self.rename_collection(info, media_list) + rename_count += len(media_list) + else: + logger.warning(f"{info.name} has no media file") + except Exception as e: + logger.warning(f"{info.name} rename failed") + logger.debug(e) + if settings.bangumi_manage.remove_bad_torrent: + self.client.delete_torrent(info.hash) + + +if __name__ == '__main__': + client = DownloadClient() + rn = Renamer(client) + rn.rename() + diff --git a/src/module/models/__init__.py b/src/module/models/__init__.py index 1cf896d6..0e595e71 100644 --- a/src/module/models/__init__.py +++ b/src/module/models/__init__.py @@ -1 +1,2 @@ from .bangumi import * +from .config import Config diff --git a/src/module/models/config.py b/src/module/models/config.py new file mode 100644 index 00000000..a4d5a6b7 --- /dev/null +++ b/src/module/models/config.py @@ -0,0 +1,69 @@ +from pydantic import BaseModel, Field + +# Sub config + + +class Program(BaseModel): + sleep_time: int = Field(7200, description="Sleep time") + rename_times: int = Field(20, description="Rename times in one loop") + webui_port: int = Field(7892, description="WebUI port") + + +class Downloader(BaseModel): + type: str = Field("qbittorrent", description="Downloader type") + host: str = Field("172.17.0.1:8080", description="Downloader host") + username: str = Field("admin", description="Downloader username") + password: str = Field("adminadmin", description="Downloader password") + path: str = Field("/downloads/Bangumi", description="Downloader path") + ssl: bool = Field(False, description="Downloader ssl") + +class RSSParser(BaseModel): + enable: bool = Field(True, description="Enable RSS parser") + type: str = Field("mikan", description="RSS parser type") + token: str = Field("token", description="RSS parser token") + custom_url: str = Field("mikanani.me", description="Custom RSS host url") + enable_tmdb: bool = Field(False, description="Enable TMDB") + filter: list[str] = Field(["720", r"\d+-\d"], description="Filter") + language: str = "zh" + + +class BangumiManage(BaseModel): + enable: bool = Field(True, description="Enable bangumi manage") + eps_complete: bool = Field(False, description="Enable eps complete") + rename_method: str = Field("pn", description="Rename method") + group_tag: bool = Field(False, description="Enable group tag") + remove_bad_torrent: bool = Field(False, description="Remove bad torrent") + + +class Debug(BaseModel): + enable: bool = Field(False, description="Enable debug") + level: str = Field("debug", description="Debug level") + file: str = Field("debug.log", description="Debug file") + dev_debug: bool = Field(False, description="Enable dev debug") + + +class Proxy(BaseModel): + enable: bool = Field(False, description="Enable proxy") + type: str = Field("http", description="Proxy type") + host: str = Field("", description="Proxy host") + port: int = Field(0, description="Proxy port") + username: str = Field("", description="Proxy username") + password: str = Field("", description="Proxy password") + + +class Notification(BaseModel): + enable: bool = Field(False, description="Enable notification") + type: str = Field("telegram", description="Notification type") + token: str = Field("", description="Notification token") + chat_id: str = Field("", description="Notification chat id") + + +class Config(BaseModel): + data_version: float = Field(4.0, description="Data version") + program: Program + downloader: Downloader + rss_parser: RSSParser + bangumi_manage: BangumiManage + debug: Debug + proxy: Proxy + notification: Notification diff --git a/src/module/network/request_url.py b/src/module/network/request_url.py index 44b53b64..1b98d905 100644 --- a/src/module/network/request_url.py +++ b/src/module/network/request_url.py @@ -43,9 +43,9 @@ class RequestURL: req.raise_for_status() return req except requests.RequestException as e: - logger.debug(f"URL: {url}") logger.debug(e) - logger.warning("ERROR with Connection.Please check DNS/Connection settings") + logger.warning(f"Cannot connect to {url}.") + logger.warning("Please check DNS/Connection settings") time.sleep(5) try_time += 1 except Exception as e: diff --git a/src/module/parser/analyser/rename_parser.py b/src/module/parser/analyser/rename_parser.py index b511f031..aa169c36 100644 --- a/src/module/parser/analyser/rename_parser.py +++ b/src/module/parser/analyser/rename_parser.py @@ -15,28 +15,42 @@ class DownloadInfo: folder_name: str +RULES = [ + r"(.*) - (\d{1,4}|\d{1,4}\.\d{1,2})(?:v\d{1,2})?(?: )?(?:END)?(.*)", + r"(.*)[\[ E](\d{1,3}|\d{1,3}\.\d{1,2})(?:v\d{1,2})?(?: )?(?:END)?[\] ](.*)", + r"(.*)\[第(\d*\.*\d*)话(?:END)?\](.*)", + r"(.*)\[第(\d*\.*\d*)話(?:END)?\](.*)", + r"(.*)第(\d*\.*\d*)话(?:END)?(.*)", + r"(.*)第(\d*\.*\d*)話(?:END)?(.*)", + r"(.*)E(\d{1,3})(.*)", +] + + class DownloadParser: def __init__(self): - self.rules = [ - r"(.*) - (\d{1,4}|\d{1,4}\.\d{1,2})(?:v\d{1,2})?(?: )?(?:END)?(.*)", - r"(.*)[\[ E](\d{1,3}|\d{1,3}\.\d{1,2})(?:v\d{1,2})?(?: )?(?:END)?[\] ](.*)", - r"(.*)\[第(\d*\.*\d*)话(?:END)?\](.*)", - r"(.*)\[第(\d*\.*\d*)話(?:END)?\](.*)", - r"(.*)第(\d*\.*\d*)话(?:END)?(.*)", - r"(.*)第(\d*\.*\d*)話(?:END)?(.*)", - ] + self.method_dict = { + "normal": self.rename_normal, + "pn": self.rename_pn, + "advance": self.rename_advance, + "no_season_pn": self.rename_no_season_pn, + "none": self.rename_none + } + @staticmethod def rename_init(name, folder_name, season, suffix) -> DownloadInfo: n = re.split(r"[\[\]()【】（）]", name) - suffix = suffix if suffix is not None else n[-1] - file_name = name.replace(f"[{n[1]}]", "") + suffix = suffix if suffix else n[-1] + if len(n) > 1: + file_name = name.replace(f"[{n[1]}]", "") + else: + file_name = name if season < 10: season = f"0{season}" return DownloadInfo(name, season, suffix, file_name, folder_name) def rename_normal(self, info: DownloadInfo): - for rule in self.rules: + for rule in RULES: match_obj = re.match(rule, info.name, re.I) if match_obj is not None: title = re.sub(r"([Ss]|Season )\d{1,3}", "", match_obj.group(1)).strip() @@ -44,7 +58,7 @@ class DownloadParser: return new_name def rename_pn(self, info: DownloadInfo): - for rule in self.rules: + for rule in RULES: match_obj = re.match(rule, info.file_name, re.I) if match_obj is not None: title = re.sub(r"([Ss]|Season )\d{1,3}", "", match_obj.group(1)).strip() @@ -57,7 +71,7 @@ class DownloadParser: return new_name def rename_advance(self, info: DownloadInfo): - for rule in self.rules: + for rule in RULES: match_obj = re.match(rule, info.file_name, re.I) if match_obj is not None: new_name = re.sub( @@ -68,7 +82,7 @@ class DownloadParser: return new_name def rename_no_season_pn(self, info: DownloadInfo): - for rule in self.rules: + for rule in RULES: match_obj = re.match(rule, info.file_name, re.I) if match_obj is not None: title = match_obj.group(1).strip() @@ -83,16 +97,16 @@ class DownloadParser: def rename_none(info: DownloadInfo): return info.name - def download_rename(self, name, folder_name, season, suffix, method): + def download_rename( + self, + name: str, + folder_name, + season, + suffix, + method + ): rename_info = self.rename_init(name, folder_name, season, suffix) - method_dict = { - "normal": self.rename_normal, - "pn": self.rename_pn, - "advance": self.rename_advance, - "no_season_pn": self.rename_no_season_pn, - "none": self.rename_none - } - return method_dict[method.lower()](rename_info) + return self.method_dict[method.lower()](rename_info) if __name__ == "__main__": diff --git a/src/module/parser/title_parser.py b/src/module/parser/title_parser.py index 72404619..ffd6f724 100644 --- a/src/module/parser/title_parser.py +++ b/src/module/parser/title_parser.py @@ -18,7 +18,14 @@ class TitleParser: def raw_parser(self, raw: str): return self._raw_parser.analyse(raw) - def download_parser(self, download_raw, folder_name, season, suffix, method=settings.bangumi_manage.method): + def download_parser( + self, + download_raw: str, + folder_name: str | None = None, + season: int | None = None, + suffix: str | None = None, + method: str = settings.bangumi_manage.rename_method + ): return self._download_parser.download_rename(download_raw, folder_name, season, suffix, method) def tmdb_parser(self, title: str, season: int):