From 65748d48082c5565920d62fb1987653e315df3f7 Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Fri, 10 Jun 2022 13:40:50 +0800 Subject: [PATCH] 2.5.0-pre --- Dockerfile | 1 + auto_bangumi/app.py | 6 +- .../{download_parser.py => rename_parser.py} | 3 + .../bangumi_parser/analyser/rss_parser.py | 4 +- auto_bangumi/bangumi_parser/fuzz_match.py | 6 +- auto_bangumi/const.py | 4 +- auto_bangumi/core/download_client.py | 14 +++-- auto_bangumi/core/download_fliter.py | 26 +++++++++ auto_bangumi/core/renamer.py | 10 +++- auto_bangumi/core/rss_collector.py | 57 ++++++++++++------- auto_bangumi/downloader/aria2_downloader.py | 38 +++++++++++++ auto_bangumi/downloader/qb_downloader.py | 3 + auto_bangumi/downloader/tr_downloader.py | 0 requirements.txt | 9 ++- resource/anidb.py | 51 +++-------------- 15 files changed, 152 insertions(+), 80 deletions(-) rename auto_bangumi/bangumi_parser/analyser/{download_parser.py => rename_parser.py} (96%) create mode 100644 auto_bangumi/core/download_fliter.py create mode 100644 auto_bangumi/downloader/aria2_downloader.py create mode 100644 auto_bangumi/downloader/tr_downloader.py diff --git a/Dockerfile b/Dockerfile index bb276c56..cfee8c25 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,3 +13,4 @@ ADD ./auto_bangumi /auto_bangumi ADD ./config /config CMD [ "python3", "app.py"] +CMD [ "uvicorn", "api:app", "--reload" ] diff --git a/auto_bangumi/app.py b/auto_bangumi/app.py index 568c9188..e270c7e8 100644 --- a/auto_bangumi/app.py +++ b/auto_bangumi/app.py @@ -74,6 +74,7 @@ def run(): quit() download_client.rss_feed() rss_collector = RSSCollector() + rename = Renamer() while True: bangumi_data = load_data_file() try: @@ -86,12 +87,15 @@ def run(): time.sleep(settings.first_sleep) bangumi_data["first_run"] = False save_data_file(bangumi_data) - Renamer(download_client).run() + if settings.method != "none": + rename.refresh() + rename.run() time.sleep(settings.sleep_time) except Exception as e: if args.debug: raise e logger.exception(e) + if __name__ == "__main__": run() diff --git a/auto_bangumi/bangumi_parser/analyser/download_parser.py b/auto_bangumi/bangumi_parser/analyser/rename_parser.py similarity index 96% rename from auto_bangumi/bangumi_parser/analyser/download_parser.py rename to auto_bangumi/bangumi_parser/analyser/rename_parser.py index 7f9cfb49..384677be 100644 --- a/auto_bangumi/bangumi_parser/analyser/download_parser.py +++ b/auto_bangumi/bangumi_parser/analyser/rename_parser.py @@ -37,6 +37,9 @@ class EPParser: ) return new_name + def rename_none(self, name): + return name + if __name__ == "__main__": name = "[NC-Raws]间谍过家家 - 09(B-Global 3840x2160 HEVC AAC MKV).mkv" diff --git a/auto_bangumi/bangumi_parser/analyser/rss_parser.py b/auto_bangumi/bangumi_parser/analyser/rss_parser.py index 22d112f8..61b40efc 100644 --- a/auto_bangumi/bangumi_parser/analyser/rss_parser.py +++ b/auto_bangumi/bangumi_parser/analyser/rss_parser.py @@ -34,6 +34,9 @@ class ParserLV2: "四": 4, "五": 5, "六": 6, + "七": 7, + "八": 8, + "九": 9, "十": 10, } name_season = re.sub(r"[\[\]]", " ", name_season) @@ -100,7 +103,6 @@ class ParserLV2: def process(self, raw_name): raw_name = self.pre_process(raw_name) - group = self.get_group(raw_name) match_obj = re.match( r"(.*|\[.*])( -? \d{1,3} |\[\d{1,3}]|\[\d{1,3}.?[vV]\d{1}]|[第第]\d{1,3}[话話集集]|\[\d{1,3}.?END])(.*)", raw_name, diff --git a/auto_bangumi/bangumi_parser/fuzz_match.py b/auto_bangumi/bangumi_parser/fuzz_match.py index 7a05890d..0d70491c 100644 --- a/auto_bangumi/bangumi_parser/fuzz_match.py +++ b/auto_bangumi/bangumi_parser/fuzz_match.py @@ -8,8 +8,8 @@ class FuzzMatch: def __init__(self, anidb_data): self.match_data = anidb_data - def match(self, title, info): - compare_value = [] + def match(self, title, info: dict): + compare_value: list = [] for type in ["main", "en", "ja", "zh-Hans", "zh-Hant"]: if info[type] is not None: a = fuzz.ratio(title.replace(" ", "").lower(), info[type].replace(" ", "").lower()) @@ -20,7 +20,7 @@ class FuzzMatch: return max(compare_value) def find_max_name(self, title): - value = [] + value: list = [] for info in self.match_data: a = self.match(title, info) value.append([a, info]) diff --git a/auto_bangumi/const.py b/auto_bangumi/const.py index 528caa7c..9135d8ed 100644 --- a/auto_bangumi/const.py +++ b/auto_bangumi/const.py @@ -5,7 +5,7 @@ DEFAULT_SETTINGS = { "sleep_time": 1800, "user_name": "admin", "password": "adminadmin", - "download_path": "/downloads/Bangumi", + "download_path": "", "method": "pn", "enable_group_tag": False, "info_path": "/config/bangumi.json", @@ -41,8 +41,6 @@ ENV_TO_ATTR = { "AB_FIRST_SLEEP": "first_sleep" } -FULL_SEASON_SUPPORT_GROUP = ["Lilith-Raws"] - class BCOLORS: @staticmethod diff --git a/auto_bangumi/core/download_client.py b/auto_bangumi/core/download_client.py index 0848ccc2..df43f602 100644 --- a/auto_bangumi/core/download_client.py +++ b/auto_bangumi/core/download_client.py @@ -26,8 +26,11 @@ class DownloadClient: "rss_refresh_interval": 30, } self.client.prefs_init(prefs=prefs) + if settings.download_path == "": + prefs = self.client.get_app_prefs() + settings.download_path = os.path.join(prefs["save_path"], "Bangumi") - def set_rule(self, bangumi_name, group, season): + def set_rule(self, bangumi_name, group, season, rss): rule = { "enable": True, "mustContain": bangumi_name, @@ -36,7 +39,7 @@ class DownloadClient: "episodeFilter": "", "smartFilter": False, "previouslyMatchedEpisodes": [], - "affectedFeeds": [settings.rss_link], + "affectedFeeds": [rss], "ignoreDays": 0, "lastMatch": "", "addPaused": settings.dev_debug, @@ -52,7 +55,6 @@ class DownloadClient: rule_name = f"[{group}] {bangumi_name}" if settings.enable_group_tag else bangumi_name self.client.rss_set_rule(rule_name=rule_name, rule_def=rule) - def rss_feed(self): try: self.client.rss_remove_item(item_path="Mikan_RSS") @@ -66,11 +68,15 @@ class DownloadClient: except ConflictError: logger.info("RSS Already exists.") + def add_collection_feed(self, rss_link): + self.client.rss_add_feed(url=rss_link) + logger.info("Add RSS Feed successfully.") + def add_rules(self, bangumi_info): logger.info("Start adding rules.") for info in bangumi_info: if not info["added"]: - self.set_rule(info["title"], info["group"], info["season"]) + self.set_rule(info["title"], info["group"], info["season"], settings.rss_link) info["added"] = True logger.info("Finished.") diff --git a/auto_bangumi/core/download_fliter.py b/auto_bangumi/core/download_fliter.py new file mode 100644 index 00000000..6a0011b1 --- /dev/null +++ b/auto_bangumi/core/download_fliter.py @@ -0,0 +1,26 @@ +import re +import logging + +from bs4 import BeautifulSoup + +from conf import settings +from utils import json_config + +logger = logging.getLogger(__name__) + + +class RSSFilter: + def __init__(self): + self.filter_rule = json_config.load(settings.filter_rule) + + def filter(self, item: BeautifulSoup): + title = item.title.string + torrent = item.find("enclosure") + download = False + for rule in self.filter_rule: + if re.search(rule["include"], title): + if not re.search(rule["exclude"], title): + download = True + logger.debug(f"{title} added") + return download, torrent + diff --git a/auto_bangumi/core/renamer.py b/auto_bangumi/core/renamer.py index 5dbdc763..e0db38b5 100644 --- a/auto_bangumi/core/renamer.py +++ b/auto_bangumi/core/renamer.py @@ -3,7 +3,7 @@ from pathlib import PurePath, PureWindowsPath from conf import settings from core.download_client import DownloadClient -from bangumi_parser.analyser.download_parser import EPParser +from bangumi_parser.analyser.rename_parser import EPParser logger = logging.getLogger(__name__) @@ -21,8 +21,14 @@ class Renamer: logger.info(f"Renamed {self.rename_count} files.") logger.info(f"Finished rename process.") + def refresh(self): + self.recent_info = self.client.get_torrent_info() + def run(self): - method_dict = {"pn": self._renamer.rename_pn, "normal": self._renamer.rename_normal} + method_dict = { + "pn": self._renamer.rename_pn, + "normal": self._renamer.rename_normal + } if settings.method not in method_dict: logger.error(f"Error method") else: diff --git a/auto_bangumi/core/rss_collector.py b/auto_bangumi/core/rss_collector.py index 64147219..1b6d9ae8 100644 --- a/auto_bangumi/core/rss_collector.py +++ b/auto_bangumi/core/rss_collector.py @@ -14,33 +14,40 @@ class RSSCollector: def __init__(self): self._simple_analyser = ParserLV2() - def collect(self, bangumi_data): + def get_rss_info(self, rss_link): try: - req = requests.get(settings.rss_link, "utf-8") + req = requests.get(rss_link, "utf-8") + rss = BeautifulSoup(req.text, "xml") + return rss except Exception as e: logger.exception(e) logger.error("ERROR with DNS/Connection.") - rss = BeautifulSoup(req.text, "xml") + + def collect(self, bangumi_data): + rss = self.get_rss_info(settings.rss_link) items = rss.find_all("item") for item in items: name = item.title.string # debug 用 if settings.debug_mode: logger.debug(f"Raw {name}") - episode = self._simple_analyser.analyse(name) - if episode: - group, title, season, ep = episode.group, episode.title, episode.season_info, episode.ep_info - sub, dpi, source = episode.subtitle, episode.dpi, episode.source - for d in bangumi_data["bangumi_info"]: - if d["title"] == title: - break - else: - if ep.number > 1 and settings.enable_eps_complete: - download_past = True - else: - download_past = False - bangumi_data["bangumi_info"].append( - { + episode, data = self.title_parser(name) + for d in bangumi_data["bangumi_info"]: + if d["title"] == episode.title: + break + bangumi_data["bangumi_info"].append(data) + logger.info(f"Adding {episode.title} Season {episode.season_info.number}") + + def title_parser(self, title): + episode = self._simple_analyser.analyse(title) + if episode: + group, title, season, ep = episode.group, episode.title, episode.season_info, episode.ep_info + sub, dpi, source = episode.subtitle, episode.dpi, episode.source + if ep.number > 1 and settings.enable_eps_complete: + download_past = True + else: + download_past = False + data = { "title": title, "season": season.raw, "group": group, @@ -50,5 +57,17 @@ class RSSCollector: "added": False, "download_past": download_past } - ) - logger.info(f"Adding {title} Season {season.number}") + return episode, data + + def collect_collection(self, rss_link): + rss = self.get_rss_info(rss_link) + item = rss.find("item") + title = item.title.string + _, data = self.title_parser(title) + return data + + +if __name__ == "__main__": + rss = RSSCollector() + data = rss.collect_collection("https://mikanani.me/RSS/Classic") + print(data) \ No newline at end of file diff --git a/auto_bangumi/downloader/aria2_downloader.py b/auto_bangumi/downloader/aria2_downloader.py new file mode 100644 index 00000000..8a4e9ec1 --- /dev/null +++ b/auto_bangumi/downloader/aria2_downloader.py @@ -0,0 +1,38 @@ +import logging +import time + +from aria2p import Client, ClientException, API + + +from conf import settings + +from downloader.exceptions import ConflictError + +logger = logging.getLogger(__name__) + + +class QbDownloader: + def __init__(self, host, username, password): + while True: + try: + self._client = API( + Client( + host=host, + port=6800, + secret=password + ) + ) + break + except ClientException: + logger.warning( + f"Can't login Aria2 Server {host} by {username}, retry in {settings.connect_retry_interval}" + ) + time.sleep(settings.connect_retry_interval) + + def torrents_add(self, urls, save_path, category): + return self._client.add_torrent( + is_paused=settings.dev_debug, + torrent_file_path=urls, + save_path=save_path, + category=category, + ) \ No newline at end of file diff --git a/auto_bangumi/downloader/qb_downloader.py b/auto_bangumi/downloader/qb_downloader.py index 397136e3..3ddf7bc5 100644 --- a/auto_bangumi/downloader/qb_downloader.py +++ b/auto_bangumi/downloader/qb_downloader.py @@ -31,6 +31,9 @@ class QbDownloader: def prefs_init(self, prefs): return self._client.app_set_preferences(prefs=prefs) + def get_app_prefs(self): + return self._client.app_preferences() + def torrents_info(self, status_filter, category): return self._client.torrents_info(status_filter, category) diff --git a/auto_bangumi/downloader/tr_downloader.py b/auto_bangumi/downloader/tr_downloader.py new file mode 100644 index 00000000..e69de29b diff --git a/requirements.txt b/requirements.txt index c99aeb2a..2e934b92 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,9 @@ qbittorrent-api -bs4 -requests +bs4~=4.11.1 +requests~=2.27.1 lxml -thefuzz +thefuzz~=0.19.0 +setuptools~=62.1.0 +fastapi~=0.78.0 +uvicorn \ No newline at end of file diff --git a/resource/anidb.py b/resource/anidb.py index 713c0eb4..15183257 100644 --- a/resource/anidb.py +++ b/resource/anidb.py @@ -29,44 +29,6 @@ def get_list(year, season): return ids -def get_title_legecy(id): - url = "https://anidb.net/anime/%s" % id - soup = BeautifulSoup(get_html(url), "lxml") - titles = soup.find("div", id="tab_2_pane") - g = titles.findAll("th") - v = titles.findAll("td") - t_dic = { - "id": id, - "main": None, - "verified": None, - "en": None, - "chs": None, - "cht": None, - "jp": None, - "synonym": None, - "kana": None - } - for i in range(0, len(g)): - if g[i].text == "Main Title": - t_dic["main"] = re.sub("\(a\d+\)", "", v[i].text).strip("\n\t") - elif g[i].text == "Official Title": - if re.search("verified", str(v[i])): - t_dic["verified"] = v[i].find("label").text - if re.search("language: english", str(v[i])): - t_dic["en"] = v[i].find("label").text - elif re.search("span>zh-Hant", str(v[i])): - t_dic["cht"] = v[i].find("label").text - elif re.search("span>zh-Hans", str(v[i])): - t_dic["chs"] = v[i].find("label").text - elif re.search("language: japanese", str(v[i])): - t_dic["jp"] = v[i].find("label").text - elif g[i].text == "Synonym": - t_dic["synonym"] = v[i].text - elif g[i].text == "Kana": - t_dic["kana"] = v[i].text - return t_dic - - def get_title(id): url = f"http://api.anidb.net:9001/httpapi?request=anime&client=autobangumi&clientver=1&protover=1&aid={id}" req = requests.get(url) @@ -97,11 +59,12 @@ def get_title(id): if __name__ == "__main__": - ids = get_list(2022, 0) json = [] - for id in ids: - data = get_title(id) - print(data) - time.sleep(2.5) - json.append(data) + for i in [0, 1, 2]: + ids = get_list(2022, i) + for id in ids: + data = get_title(id) + print(data) + time.sleep(2.5) + json.append(data) json_config.save("season_winter.json", json)