From 21734a2ed38dc80583d451363e06481d3fe925b2 Mon Sep 17 00:00:00 2001 From: Bluefissure Date: Tue, 7 Mar 2023 16:28:51 -0600 Subject: [PATCH 01/30] fix: attr as key in settings tuple attr cannot be key (causing docker failing hard when creating the config file) --- module/conf/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/conf/config.py b/module/conf/config.py index 71cda65f..462e1e93 100644 --- a/module/conf/config.py +++ b/module/conf/config.py @@ -52,7 +52,7 @@ class Settings: for key, section in ENV_TO_ATTR.items(): for env, attr in section.items(): if env in os.environ: - settings[key][attr] = self._val_from_env(env, attr) + settings[key][env] = self._val_from_env(env, attr) with open(CONFIG_PATH, "w") as f: json.dump(settings, f, indent=4) return settings From 4871dbe14c0164cb73260b0db6843c010f83c635 Mon Sep 17 00:00:00 2001 From: Bluefissure Date: Tue, 7 Mar 2023 16:38:47 -0600 Subject: [PATCH 02/30] Update config.py --- module/conf/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/module/conf/config.py b/module/conf/config.py index 462e1e93..56a88c43 100644 --- a/module/conf/config.py +++ b/module/conf/config.py @@ -52,7 +52,8 @@ class Settings: for key, section in ENV_TO_ATTR.items(): for env, attr in section.items(): if env in os.environ: - settings[key][env] = self._val_from_env(env, attr) + attr_name = attr[0] if isinstance(attr, tuple) else attr + settings[key][attr_name] = self._val_from_env(env, attr) with open(CONFIG_PATH, "w") as f: json.dump(settings, f, indent=4) return settings From 219fe0cd17ae90f75d9ee8151ccce7a657aa8971 Mon Sep 17 00:00:00 2001 From: Bluefissure Date: Tue, 7 Mar 2023 22:43:37 +0000 Subject: [PATCH 03/30] fix: e.lower() will never be 'True' --- module/conf/const.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/module/conf/const.py b/module/conf/const.py index 14f2cb16..0a8fae7e 100644 --- a/module/conf/const.py +++ b/module/conf/const.py @@ -66,21 +66,21 @@ ENV_TO_ATTR = { "AB_DOWNLOAD_PATH": "path", }, "rss_parser": { - "AB_RSS_COLLECTOR": ("enable", lambda e: e.lower() in ("True", "1", "t")), + "AB_RSS_COLLECTOR": ("enable", lambda e: e.lower() in ("true", "1", "t")), "AB_RSS": "link", "AB_NOT_CONTAIN": ("filter", lambda e: e.split("|")), "AB_LANGUAGE": "language", - "AB_ENABLE_TMDB": ("enable_tmdb", lambda e: e.lower() in ("True", "1", "t")), + "AB_ENABLE_TMDB": ("enable_tmdb", lambda e: e.lower() in ("true", "1", "t")), }, "bangumi_manage": { - "AB_RENAME": ("enable", lambda e: e.lower() in ("True", "1", "t")), + "AB_RENAME": ("enable", lambda e: e.lower() in ("true", "1", "t")), "AB_METHOD": "method", - "AB_GROUP_TAG": ("group_tag", lambda e: e.lower() in ("True", "1", "t")), - "AB_EP_COMPLETE": ("eps_complete", lambda e: e.lower() in ("True", "1", "t")), - "AB_REMOVE_BAD_BT": ("remove_bad_torrent", lambda e: e.lower() in ("True", "1", "t")), + "AB_GROUP_TAG": ("group_tag", lambda e: e.lower() in ("true", "1", "t")), + "AB_EP_COMPLETE": ("eps_complete", lambda e: e.lower() in ("true", "1", "t")), + "AB_REMOVE_BAD_BT": ("remove_bad_torrent", lambda e: e.lower() in ("true", "1", "t")), }, "debug": { - "AB_DEBUG_MODE": ("enable", lambda e: e.lower() in ("True", "1", "t")), + "AB_DEBUG_MODE": ("enable", lambda e: e.lower() in ("true", "1", "t")), }, "proxy": { "AB_HTTP_PROXY": "http", From 7e45230d1b446f4078517025c5c550d7abd087c0 Mon Sep 17 00:00:00 2001 From: WeijiangChen Date: Sun, 12 Mar 2023 21:51:03 +0800 Subject: [PATCH 04/30] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BA=86=E8=AF=B7?= =?UTF-8?q?=E6=B1=82=E7=B1=BB=EF=BC=8C=E9=81=BF=E5=85=8D=E8=AF=B7=E6=B1=82?= =?UTF-8?q?=E5=BF=98=E8=AE=B0=E5=85=B3=EF=BC=8C=E8=AF=B7=E6=B1=82=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5=E9=87=8D=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- module/core/eps_complete.py | 5 ++- module/core/rss_analyser.py | 9 ++--- module/network/request_url.py | 57 +++++++++++++++------------ module/parser/analyser/bgm_parser.py | 10 ++--- module/parser/analyser/tmdb_parser.py | 38 +++++++++--------- 5 files changed, 63 insertions(+), 56 deletions(-) diff --git a/module/core/eps_complete.py b/module/core/eps_complete.py index 2954d5cf..6b1ede72 100644 --- a/module/core/eps_complete.py +++ b/module/core/eps_complete.py @@ -13,7 +13,7 @@ SEARCH_KEY = ["group", "title_raw", "season_raw", "subtitle", "source", "dpi"] class FullSeasonGet: def __init__(self): - self._get_rss = RequestContent() + pass @staticmethod def init_eps_complete_search_str(data: dict): @@ -24,7 +24,8 @@ class FullSeasonGet: def get_season_torrents(self, data: dict): keyword = self.init_eps_complete_search_str(data) - torrents = self._get_rss.get_torrents(f"https://mikanani.me/RSS/Search?searchstr={keyword}") + with RequestContent() as req: + torrents = req.get_torrents(f"https://mikanani.me/RSS/Search?searchstr={keyword}") return torrents @staticmethod diff --git a/module/core/rss_analyser.py b/module/core/rss_analyser.py index 55a04e64..aa611e30 100644 --- a/module/core/rss_analyser.py +++ b/module/core/rss_analyser.py @@ -14,11 +14,10 @@ logger = logging.getLogger(__name__) class RSSAnalyser: def __init__(self): self._title_analyser = TitleParser() - self._request = RequestContent() def rss_to_datas(self, bangumi_info: list) -> list: - rss_torrents = self._request.get_torrents(settings.rss_parser.link) - self._request.close_session() + with RequestContent() as req: + rss_torrents = req.get_torrents(settings.rss_parser.link) for torrent in rss_torrents: raw_title = torrent.name extra_add = True @@ -35,8 +34,8 @@ class RSSAnalyser: return bangumi_info def rss_to_data(self, url) -> dict: - rss_torrents = self._request.get_torrents(url) - self._request.close_session() + with RequestContent() as req: + rss_torrents = req.get_torrents(url) for torrent in rss_torrents: try: data = self._title_analyser.return_dict(torrent.name) diff --git a/module/network/request_url.py b/module/network/request_url.py index bedc6225..4f38734b 100644 --- a/module/network/request_url.py +++ b/module/network/request_url.py @@ -14,7 +14,37 @@ logger = logging.getLogger(__name__) class RequestURL: def __init__(self): - self.session = requests.session() + self.header = { + "user-agent": "Mozilla/5.0", + "Accept": "application/xml" + } + + def get_url(self, url): + times = 0 + while times < 5: + try: + req = self.session.get(url=url, headers=self.header) + req.raise_for_status() + return req + except requests.RequestException as e: + logger.debug(f"URL: {url}") + logger.debug(e) + logger.warning("ERROR with Connection.Please check DNS/Connection settings") + time.sleep(5) + times += 1 + except Exception as e: + logger.debug(f"URL: {url}") + logger.debug(e) + break + + def get_content(self, url, content="xml"): + if content == "xml": + return BeautifulSoup(self.get_url(url).text, content) + elif content == "json": + return self.get_url(url).json() + + def __enter__(self): + self.session = requests.Session() if settings.proxy.enable: if settings.proxy.type == "http": url = f"http://{settings.proxy.host}:{settings.proxy.port}" @@ -26,31 +56,8 @@ class RequestURL: socks.set_default_proxy(socks.SOCKS5, addr=settings.proxy.host, port=settings.proxy.port, rdns=True, username=settings.proxy.username, password=settings.proxy.password) socket.socket = socks.socksocket - self.header = { - "user-agent": "Mozilla/5.0", - "Accept": "application/xml" - } - def get_url(self, url): - times = 0 - while times < 5: - try: - req = self.session.get(url=url, headers=self.header) - return req - except Exception as e: - logger.debug(f"URL: {url}") - logger.debug(e) - logger.warning("ERROR with Connection.Please check DNS/Connection settings") - time.sleep(5) - times += 1 - - def get_content(self, url, content="xml"): - if content == "xml": - return BeautifulSoup(self.get_url(url).text, content) - elif content == "json": - return self.get_url(url).json() - - def close(self): + def __exit__(self, exc_type, exc_val, exc_tb): self.session.close() diff --git a/module/parser/analyser/bgm_parser.py b/module/parser/analyser/bgm_parser.py index acd7eeb1..427cda11 100644 --- a/module/parser/analyser/bgm_parser.py +++ b/module/parser/analyser/bgm_parser.py @@ -7,11 +7,11 @@ class BgmAPI: f"https://api.bgm.tv/search/subject/{e}?type=2" self.info_url = lambda e: \ f"https://api.bgm.tv/subject/{e}" - self._request = RequestContent() def search(self, title): url = self.search_url(title) - contents = self._request.get_json(url)["list"] - if contents.__len__() == 0: - return None - return contents[0]["name"], contents[0]["name_cn"] \ No newline at end of file + with RequestContent() as req: + contents = req.get_json(url)["list"] + if contents.__len__() == 0: + return None + return contents[0]["name"], contents[0]["name_cn"] diff --git a/module/parser/analyser/tmdb_parser.py b/module/parser/analyser/tmdb_parser.py index 9765ed70..a5759344 100644 --- a/module/parser/analyser/tmdb_parser.py +++ b/module/parser/analyser/tmdb_parser.py @@ -22,14 +22,14 @@ class TMDBMatcher: f"https://api.themoviedb.org/3/search/tv?api_key={TMDB_API}&page=1&query={e}&include_adult=false" self.info_url = lambda e: \ f"https://api.themoviedb.org/3/tv/{e}?api_key={TMDB_API}&language=zh-CN" - self._request = RequestContent() def is_animation(self, tv_id) -> bool: url_info = self.info_url(tv_id) - type_id = self._request.get_json(url_info)["genres"] - for type in type_id: - if type.get("id") == 16: - return True + with RequestContent() as req: + type_id = req.get_json(url_info)["genres"] + for type in type_id: + if type.get("id") == 16: + return True return False # def get_zh_title(self, id): @@ -51,20 +51,20 @@ class TMDBMatcher: return int(re.findall(r"\d", season.get("season"))[0]) def tmdb_search(self, title) -> TMDBInfo: - url = self.search_url(title) - contents = self._request.get_json(url).get("results") - if contents.__len__() == 0: - url = self.search_url(title.replace(" ", "")) - contents = self._request.get_json(url).get("results") - # 判断动画 - for content in contents: - id = content["id"] - if self.is_animation(id): - break - url_info = self.info_url(id) - info_content = self._request.get_json(url_info) - # 关闭链接 - self._request.close() + with RequestContent() as req: + url = self.search_url(title) + contents = req.get_json(url).get("results") + if contents.__len__() == 0: + url = self.search_url(title.replace(" ", "")) + contents = req.get_json(url).get("results") + # 判断动画 + for content in contents: + id = content["id"] + if self.is_animation(id): + break + url_info = self.info_url(id) + info_content = req.get_json(url_info) + season = [{"season": s.get("name"), "air_date": s.get("air_date")} for s in info_content.get("seasons")] last_season = self.get_season(season) title_jp = info_content.get("original_name") From ec56f58e4765b43fe4c1747fff1fec592c30e5c5 Mon Sep 17 00:00:00 2001 From: WeijiangChen Date: Sun, 12 Mar 2023 21:56:04 +0800 Subject: [PATCH 05/30] =?UTF-8?q?=E5=BF=98=E8=AE=B0=E9=87=8D=E8=BD=BD?= =?UTF-8?q?=E4=BA=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- module/network/request_contents.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/module/network/request_contents.py b/module/network/request_contents.py index 631c32ca..cb689ea7 100644 --- a/module/network/request_contents.py +++ b/module/network/request_contents.py @@ -7,16 +7,14 @@ import re FILTER = "|".join(settings.rss_parser.filter) + @dataclass class TorrentInfo: name: str torrent_link: str -class RequestContent: - def __init__(self): - self._req = RequestURL() - +class RequestContent(RequestURL): # Mikanani RSS def get_torrents(self, _url: str) -> [TorrentInfo]: soup = self._req.get_content(_url) @@ -37,6 +35,3 @@ class RequestContent: # API JSON def get_json(self, _url) -> dict: return self._req.get_content(_url, content="json") - - def close_session(self): - self._req.close() From 106e4847c0634cda87b5d710eda47fc7881a02ae Mon Sep 17 00:00:00 2001 From: WeijiangChen Date: Sun, 12 Mar 2023 22:34:47 +0800 Subject: [PATCH 06/30] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E7=B1=BB=E9=87=8D?= =?UTF-8?q?=E8=BD=BD=E5=BF=98=E6=94=B9=E5=87=BD=E6=95=B0=E4=BA=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- module/network/request_contents.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/module/network/request_contents.py b/module/network/request_contents.py index cb689ea7..15b04c60 100644 --- a/module/network/request_contents.py +++ b/module/network/request_contents.py @@ -17,7 +17,7 @@ class TorrentInfo: class RequestContent(RequestURL): # Mikanani RSS def get_torrents(self, _url: str) -> [TorrentInfo]: - soup = self._req.get_content(_url) + soup = self.session.get_content(_url) torrent_titles = [item.title.string for item in soup.find_all("item")] torrent_urls = [item.get("url") for item in soup.find_all("enclosure")] torrents = [] @@ -27,11 +27,11 @@ class RequestContent(RequestURL): return torrents def get_torrent(self, _url) -> TorrentInfo: - soup = self._req.get_content(_url) + soup = self.session.get_content(_url) item = soup.find("item") enclosure = item.find("enclosure") return TorrentInfo(item.title.string, enclosure["url"]) # API JSON def get_json(self, _url) -> dict: - return self._req.get_content(_url, content="json") + return self.session.get_content(_url, content="json") From fa944a34b39ff77ecb79cfae079365ee2220d494 Mon Sep 17 00:00:00 2001 From: WeijiangChen Date: Sun, 12 Mar 2023 22:35:25 +0800 Subject: [PATCH 07/30] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E7=B1=BB=E9=87=8D?= =?UTF-8?q?=E8=BD=BD=E5=BF=98=E6=94=B9=E5=87=BD=E6=95=B0=E4=BA=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- module/network/request_contents.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/module/network/request_contents.py b/module/network/request_contents.py index 15b04c60..3c9506c8 100644 --- a/module/network/request_contents.py +++ b/module/network/request_contents.py @@ -17,7 +17,7 @@ class TorrentInfo: class RequestContent(RequestURL): # Mikanani RSS def get_torrents(self, _url: str) -> [TorrentInfo]: - soup = self.session.get_content(_url) + soup = self.get_content(_url) torrent_titles = [item.title.string for item in soup.find_all("item")] torrent_urls = [item.get("url") for item in soup.find_all("enclosure")] torrents = [] @@ -27,11 +27,11 @@ class RequestContent(RequestURL): return torrents def get_torrent(self, _url) -> TorrentInfo: - soup = self.session.get_content(_url) + soup = self.get_content(_url) item = soup.find("item") enclosure = item.find("enclosure") return TorrentInfo(item.title.string, enclosure["url"]) # API JSON def get_json(self, _url) -> dict: - return self.session.get_content(_url, content="json") + return self.get_content(_url, content="json") From aee3a3be3312fd10c3552fe6a0354a7e00b62d1c Mon Sep 17 00:00:00 2001 From: WeijiangChen Date: Sun, 12 Mar 2023 22:43:06 +0800 Subject: [PATCH 08/30] =?UTF-8?q?=E6=98=8E=E7=A1=AEget=20json=E4=B8=8Eget?= =?UTF-8?q?=20xml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- module/network/request_contents.py | 11 ++++++++--- module/network/request_url.py | 8 -------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/module/network/request_contents.py b/module/network/request_contents.py index 3c9506c8..6a0bfbf6 100644 --- a/module/network/request_contents.py +++ b/module/network/request_contents.py @@ -1,5 +1,7 @@ from dataclasses import dataclass +from bs4 import BeautifulSoup + from .request_url import RequestURL from module.conf import settings @@ -17,7 +19,7 @@ class TorrentInfo: class RequestContent(RequestURL): # Mikanani RSS def get_torrents(self, _url: str) -> [TorrentInfo]: - soup = self.get_content(_url) + soup = self.get_xml(_url) torrent_titles = [item.title.string for item in soup.find_all("item")] torrent_urls = [item.get("url") for item in soup.find_all("enclosure")] torrents = [] @@ -27,11 +29,14 @@ class RequestContent(RequestURL): return torrents def get_torrent(self, _url) -> TorrentInfo: - soup = self.get_content(_url) + soup = self.get_xml(_url) item = soup.find("item") enclosure = item.find("enclosure") return TorrentInfo(item.title.string, enclosure["url"]) + def get_xml(self, url): + return BeautifulSoup(self.get_url(url).text, "xml") + # API JSON def get_json(self, _url) -> dict: - return self.get_content(_url, content="json") + return self.get_url(_url).json() diff --git a/module/network/request_url.py b/module/network/request_url.py index 4f38734b..5dbd30e6 100644 --- a/module/network/request_url.py +++ b/module/network/request_url.py @@ -5,8 +5,6 @@ import socket import socks import logging -from bs4 import BeautifulSoup - from module.conf import settings logger = logging.getLogger(__name__) @@ -37,12 +35,6 @@ class RequestURL: logger.debug(e) break - def get_content(self, url, content="xml"): - if content == "xml": - return BeautifulSoup(self.get_url(url).text, content) - elif content == "json": - return self.get_url(url).json() - def __enter__(self): self.session = requests.Session() if settings.proxy.enable: From f4d4a06f371d431dbefb8fcb65ecaca5e69fc152 Mon Sep 17 00:00:00 2001 From: WeijiangChen Date: Sun, 12 Mar 2023 22:45:55 +0800 Subject: [PATCH 09/30] =?UTF-8?q?=E6=BC=8F=E6=94=B9=E4=BA=86=E4=B8=80?= =?UTF-8?q?=E4=B8=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- module/core/eps_complete.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/module/core/eps_complete.py b/module/core/eps_complete.py index 6b1ede72..87c2a3c9 100644 --- a/module/core/eps_complete.py +++ b/module/core/eps_complete.py @@ -57,7 +57,8 @@ class FullSeasonGet: self.download_eps(data, download_client) def download_collection(self, data, link, download_client: DownloadClient): - torrents = self._get_rss.get_torrents(link) + with RequestContent() as req: + torrents = req.get_torrents(link) downloads = self.collect_season_torrents(data, torrents) logger.info(f"Starting download {data.get('official_title')}") for download in downloads: From e9f493863b0efc2bf8c51c2b268afcebc17df703 Mon Sep 17 00:00:00 2001 From: WeijiangChen Date: Mon, 13 Mar 2023 00:17:30 +0800 Subject: [PATCH 10/30] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=8E=A8=E9=80=81?= =?UTF-8?q?=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- module/core/renamer.py | 3 +++ module/network/__init__.py | 6 +----- module/network/notification.py | 34 +++++++++++++++++++++++++++++++++- 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/module/core/renamer.py b/module/core/renamer.py index 98d5c2e3..bd31f99e 100644 --- a/module/core/renamer.py +++ b/module/core/renamer.py @@ -8,6 +8,7 @@ from .download_client import DownloadClient from module.conf import settings from module.parser import TitleParser +from ..network import PostNotification, FtqqNotification logger = logging.getLogger(__name__) @@ -53,6 +54,7 @@ class Renamer: return path_name, season, folder_name, suffix, download_path def run(self): + notification = FtqqNotification() recent_info, torrent_count = self.get_torrent_info() rename_count = 0 for info in recent_info: @@ -68,6 +70,7 @@ class Renamer: old_name = info.content_path.replace(info.save_path, "") self.client.rename_torrent_file(torrent_hash, new_name, old_name, new_name) rename_count += 1 + notification.send_msg(f"《{name[:10]}》缓存成功", f"[Auto Bangumi]《{name}》缓存成功") else: continue except Exception as e: diff --git a/module/network/__init__.py b/module/network/__init__.py index 269d54ac..326efdf1 100644 --- a/module/network/__init__.py +++ b/module/network/__init__.py @@ -1,6 +1,2 @@ from .request_contents import RequestContent -from .notification import PostNotification - - - - +from .notification import PostNotification, FtqqNotification diff --git a/module/network/notification.py b/module/network/notification.py index 06086f0b..75dc141c 100644 --- a/module/network/notification.py +++ b/module/network/notification.py @@ -1,8 +1,13 @@ +import logging + import requests from module.conf import settings +logger = logging.getLogger(__name__) + + class PostNotification: def __init__(self): self.token = settings.notification_token @@ -11,4 +16,31 @@ class PostNotification: def ifttt_post(self, message): url = self.notification_url(message) response = requests.get(url) - return response.status_code == 200 \ No newline at end of file + return response.status_code == 200 + + +class FtqqNotification: + """Server酱推送""" + + def __init__(self): + self.token = settings.notification.token + self.notification_url = f"https://sctapi.ftqq.com/{self.token}.send" + + def send_msg(self, title: str, desp: str) -> bool: + data = { + "title": title, + "desp": desp, + } + try: + resp = requests.post(self.notification_url, json=data, timeout=3) + resp.raise_for_status() + except requests.RequestException as e: + logging.error("[FtqqNotification] send fail, error: %s" % e) + return False + return True + + +if __name__ == '__main__': + name = "勇者、辞职不干了" + notification = FtqqNotification() + notification.send_msg(f"《{name[:10]}》缓存成功", f"[Auto Bangumi]《{name}》缓存成功") From dc8347ffaaa247b162e8044cb8acc047896f8bd0 Mon Sep 17 00:00:00 2001 From: WeijiangChen Date: Mon, 13 Mar 2023 00:28:40 +0800 Subject: [PATCH 11/30] =?UTF-8?q?=E9=87=8D=E5=91=BD=E5=90=8D=E6=8E=A8?= =?UTF-8?q?=E9=80=81=E5=87=BD=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- module/core/renamer.py | 4 ++-- module/network/__init__.py | 2 +- module/network/notification.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/module/core/renamer.py b/module/core/renamer.py index bd31f99e..7cd681f2 100644 --- a/module/core/renamer.py +++ b/module/core/renamer.py @@ -8,7 +8,7 @@ from .download_client import DownloadClient from module.conf import settings from module.parser import TitleParser -from ..network import PostNotification, FtqqNotification +from ..network import PostNotification, ServerChanNotification logger = logging.getLogger(__name__) @@ -54,7 +54,7 @@ class Renamer: return path_name, season, folder_name, suffix, download_path def run(self): - notification = FtqqNotification() + notification = ServerChanNotification() recent_info, torrent_count = self.get_torrent_info() rename_count = 0 for info in recent_info: diff --git a/module/network/__init__.py b/module/network/__init__.py index 326efdf1..d4589016 100644 --- a/module/network/__init__.py +++ b/module/network/__init__.py @@ -1,2 +1,2 @@ from .request_contents import RequestContent -from .notification import PostNotification, FtqqNotification +from .notification import PostNotification, ServerChanNotification diff --git a/module/network/notification.py b/module/network/notification.py index 75dc141c..1c0bb119 100644 --- a/module/network/notification.py +++ b/module/network/notification.py @@ -19,7 +19,7 @@ class PostNotification: return response.status_code == 200 -class FtqqNotification: +class ServerChanNotification: """Server酱推送""" def __init__(self): @@ -35,12 +35,12 @@ class FtqqNotification: resp = requests.post(self.notification_url, json=data, timeout=3) resp.raise_for_status() except requests.RequestException as e: - logging.error("[FtqqNotification] send fail, error: %s" % e) + logging.error("[ServerChanNotification] send fail, error: %s" % e) return False return True if __name__ == '__main__': name = "勇者、辞职不干了" - notification = FtqqNotification() + notification = ServerChanNotification() notification.send_msg(f"《{name[:10]}》缓存成功", f"[Auto Bangumi]《{name}》缓存成功") From b901319fb151efee9fb30692f7ada7818246177e Mon Sep 17 00:00:00 2001 From: WeijiangChen Date: Mon, 13 Mar 2023 20:50:59 +0800 Subject: [PATCH 12/30] =?UTF-8?q?=E5=A6=82=E6=9E=9C=E9=80=9A=E7=9F=A5?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=E6=B2=A1=E5=BC=80=E5=90=AF=EF=BC=8C=E4=B8=8D?= =?UTF-8?q?=E5=90=AF=E7=94=A8=E9=80=9A=E7=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- module/network/notification.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/module/network/notification.py b/module/network/notification.py index 1c0bb119..834f9a68 100644 --- a/module/network/notification.py +++ b/module/network/notification.py @@ -27,6 +27,8 @@ class ServerChanNotification: self.notification_url = f"https://sctapi.ftqq.com/{self.token}.send" def send_msg(self, title: str, desp: str) -> bool: + if not settings.notification.enable: + return False data = { "title": title, "desp": desp, From 2745047924e9de7642755809081722ebc1286620 Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Mon, 13 Mar 2023 23:31:12 +0800 Subject: [PATCH 13/30] Update Dockerfile, Fix bugs --- .github/workflows/dev-latest.yml | 4 ++-- Dockerfile | 20 ++++++++++++++------ module/api.py | 13 +------------ module/app.py | 3 --- module/models/__init__.py | 0 module/models/api.py | 18 ++++++++++++++++++ module/models/bangumi.py | 14 ++++++++++++++ 7 files changed, 49 insertions(+), 23 deletions(-) create mode 100644 module/models/__init__.py create mode 100644 module/models/api.py create mode 100644 module/models/bangumi.py diff --git a/.github/workflows/dev-latest.yml b/.github/workflows/dev-latest.yml index 1803e876..2071149a 100644 --- a/.github/workflows/dev-latest.yml +++ b/.github/workflows/dev-latest.yml @@ -12,7 +12,7 @@ jobs: uses: actions/checkout@v3 - name: Create Version info run: | - echo "version='2.6.0'" > module/conf/version.py + echo "VERSION = '2.6.0-beta'" > module/conf/version.py - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -30,5 +30,5 @@ jobs: uses: docker/build-push-action@v3 with: push: true - tags: estrellaxd/auto_bangumi:dev-latest + tags: estrellaxd/auto_bangumi:2.6.0-beta2 file: ./Dockerfile diff --git a/Dockerfile b/Dockerfile index 6882dbae..9035d817 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,17 @@ # syntax=docker/dockerfile:1 -FROM python:3.11-buster AS build +FROM python:3.11-alpine AS build RUN mkdir /install WORKDIR /install COPY requirements.txt . + +RUN apk update \ + && apk add --no-cache \ + curl \ + gcc \ + g++ \ + linux-headers \ + RUN python3 -m pip install --upgrade pip \ && pip install -r requirements.txt --prefix="/install" @@ -14,10 +22,10 @@ ENV TZ=Asia/Shanghai \ PGID=1000 \ UMASK=022 -WORKDIR / +WORKDIR /app COPY --from=build --chmod=777 /install /usr/local -COPY --chmod=755 . . +COPY --chmod=755 . /app RUN apk add --no-cache \ curl \ @@ -33,13 +41,13 @@ RUN wget "https://github.com/Rewrite0/Auto_Bangumi_WebUI/releases/download/$(cur RUN addgroup -S auto_bangumi -g 1000 && \ adduser -S auto_bangumi -G auto_bangumi -h /home/auto_bangumi -u 1000 && \ usermod -s /bin/bash auto_bangumi && \ - mkdir -p "/config" && \ - mkdir -p "/data" && \ + mkdir -p "config" && \ + mkdir -p "data" && \ chmod a+x \ run.sh EXPOSE 7892 -VOLUME [ "/config" , "/data"] +VOLUME [ "config" , "data"] CMD ["sh", "run.sh"] diff --git a/module/api.py b/module/api.py index e434be66..1d6fdb83 100644 --- a/module/api.py +++ b/module/api.py @@ -4,12 +4,12 @@ from fastapi import FastAPI, Request from fastapi.responses import HTMLResponse, FileResponse from fastapi.templating import Jinja2Templates from fastapi.staticfiles import StaticFiles -from pydantic import BaseModel import logging from .core import APIProcess from .conf import settings, DATA_PATH, LOG_PATH from .utils import json_config +from .models.api import * logger = logging.getLogger(__name__) @@ -47,10 +47,6 @@ def remove_rule(bangumi_title: str): return api_func.remove_rule(bangumi_title) -class RssLink(BaseModel): - rss_link: str - - @app.post("/api/v1/collection") async def collection(link: RssLink): return api_func.download_collection(link.rss_link) @@ -61,11 +57,6 @@ async def subscribe(link: RssLink): return api_func.add_subscribe(link.rss_link) -class AddRule(BaseModel): - title: str - season: int - - @app.post("/api/v1/addRule") async def add_rule(info: AddRule): return api_func.add_rule(info.title, info.season) @@ -76,6 +67,4 @@ def run(): uvicorn.run(app, host="0.0.0.0", port=settings.program.webui_port) -if __name__ == "__main__": - run() diff --git a/module/app.py b/module/app.py index 1aafb0c2..9d10ea1c 100644 --- a/module/app.py +++ b/module/app.py @@ -87,6 +87,3 @@ def run(): bangumi_data = load_data_file() # 主程序循环 main_process(bangumi_data, download_client) - -if __name__ == '__main__': - run() \ No newline at end of file diff --git a/module/models/__init__.py b/module/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/module/models/api.py b/module/models/api.py new file mode 100644 index 00000000..9843e7cb --- /dev/null +++ b/module/models/api.py @@ -0,0 +1,18 @@ +from pydantic import BaseModel + + +class RssLink(BaseModel): + rss_link: str + + +class AddRule(BaseModel): + title: str + season: int + + +class ChangeConfig(BaseModel): + config: dict + + +class ChangeRule(BaseModel): + rule: dict \ No newline at end of file diff --git a/module/models/bangumi.py b/module/models/bangumi.py new file mode 100644 index 00000000..c175bc10 --- /dev/null +++ b/module/models/bangumi.py @@ -0,0 +1,14 @@ +from dataclasses import dataclass + + +@dataclass +class MatchRule: + keyword: str + filter: list + rss_link: str + + +class GroupFilter: + name: str + filter: list + From 34216245f75552eb7201be532448604362f4f6e8 Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Tue, 14 Mar 2023 21:22:55 +0800 Subject: [PATCH 14/30] refactor - fix rename bug - temp save --- .dockerignore | 4 +-- .github/workflows/dev-latest.yml | 2 +- .github/workflows/docker.yml | 2 +- .github/workflows/unittest.yml | 9 ++++--- .gitignore | 22 +++++++-------- Dockerfile | 2 +- main.py | 16 ----------- module/core/__init__.py | 5 ---- setup.py | 27 ------------------- src/main.py | 18 +++++++++++++ {module => src/module}/__init__.py | 0 .../module}/ab_decorator/__init__.py | 0 {module => src/module}/api.py | 0 {module => src/module}/app.py | 9 ++++--- {module => src/module}/conf/__init__.py | 2 +- {module => src/module}/conf/config.py | 20 +++++++------- {module => src/module}/conf/const.py | 0 {module => src/module}/conf/log.py | 3 ++- {module => src/module}/conf/logging.conf | 0 {module => src/module}/conf/parse.py | 0 src/module/core/__init__.py | 2 ++ {module => src/module}/core/api_func.py | 4 ++- .../module}/core/download_client.py | 6 ----- .../module}/core/download_fliter.py | 0 {module => src/module}/downloader/__init__.py | 4 +-- .../module}/downloader/aria2_downloader.py | 0 .../module}/downloader/exceptions.py | 0 .../module}/downloader/qb_downloader.py | 0 .../module}/downloader/tr_downloader.py | 0 src/module/manager/__init__.py | 2 ++ .../module/manager}/eps_complete.py | 2 +- .../core => src/module/manager}/renamer.py | 9 +++---- {module/core => src/module/manager}/repath.py | 0 {module => src/module}/models/__init__.py | 0 {module => src/module}/models/api.py | 0 {module => src/module}/models/bangumi.py | 0 {module => src/module}/network/__init__.py | 0 .../module}/network/notification.py | 14 ++++++++++ .../module}/network/request_contents.py | 1 + {module => src/module}/network/request_url.py | 0 {module => src/module}/parser/__init__.py | 0 .../module}/parser/analyser/__init__.py | 0 .../module}/parser/analyser/bgm_parser.py | 0 .../module}/parser/analyser/raw_parser.py | 0 .../module}/parser/analyser/rename_parser.py | 0 .../module}/parser/analyser/tmdb_parser.py | 0 {module => src/module}/parser/fuzz_match.py | 0 {module => src/module}/parser/title_parser.py | 0 src/module/rss/__init__.py | 1 + .../core => src/module/rss}/rss_analyser.py | 1 - {module => src/module}/setID.sh | 0 {module => src/module}/utils/__init__.py | 0 {module => src/module}/utils/json_config.py | 0 run.sh => src/run.sh | 0 {test => src/test}/__init__.py | 0 {test => src/test}/test_raw_parser.py | 0 56 files changed, 86 insertions(+), 101 deletions(-) delete mode 100644 main.py delete mode 100644 module/core/__init__.py delete mode 100644 setup.py create mode 100644 src/main.py rename {module => src/module}/__init__.py (100%) rename {module => src/module}/ab_decorator/__init__.py (100%) rename {module => src/module}/api.py (100%) rename {module => src/module}/app.py (92%) rename {module => src/module}/conf/__init__.py (100%) rename {module => src/module}/conf/config.py (86%) rename {module => src/module}/conf/const.py (100%) rename {module => src/module}/conf/log.py (94%) rename {module => src/module}/conf/logging.conf (100%) rename {module => src/module}/conf/parse.py (100%) create mode 100644 src/module/core/__init__.py rename {module => src/module}/core/api_func.py (94%) rename {module => src/module}/core/download_client.py (96%) rename {module => src/module}/core/download_fliter.py (100%) rename {module => src/module}/downloader/__init__.py (71%) rename {module => src/module}/downloader/aria2_downloader.py (100%) rename {module => src/module}/downloader/exceptions.py (100%) rename {module => src/module}/downloader/qb_downloader.py (100%) rename {module => src/module}/downloader/tr_downloader.py (100%) create mode 100644 src/module/manager/__init__.py rename {module/core => src/module/manager}/eps_complete.py (98%) rename {module/core => src/module/manager}/renamer.py (90%) rename {module/core => src/module/manager}/repath.py (100%) rename {module => src/module}/models/__init__.py (100%) rename {module => src/module}/models/api.py (100%) rename {module => src/module}/models/bangumi.py (100%) rename {module => src/module}/network/__init__.py (100%) rename {module => src/module}/network/notification.py (76%) rename {module => src/module}/network/request_contents.py (99%) rename {module => src/module}/network/request_url.py (100%) rename {module => src/module}/parser/__init__.py (100%) rename {module => src/module}/parser/analyser/__init__.py (100%) rename {module => src/module}/parser/analyser/bgm_parser.py (100%) rename {module => src/module}/parser/analyser/raw_parser.py (100%) rename {module => src/module}/parser/analyser/rename_parser.py (100%) rename {module => src/module}/parser/analyser/tmdb_parser.py (100%) rename {module => src/module}/parser/fuzz_match.py (100%) rename {module => src/module}/parser/title_parser.py (100%) create mode 100644 src/module/rss/__init__.py rename {module/core => src/module/rss}/rss_analyser.py (99%) rename {module => src/module}/setID.sh (100%) rename {module => src/module}/utils/__init__.py (100%) rename {module => src/module}/utils/json_config.py (100%) rename run.sh => src/run.sh (100%) rename {test => src/test}/__init__.py (100%) rename {test => src/test}/test_raw_parser.py (100%) diff --git a/.dockerignore b/.dockerignore index 3d5cb7ed..e597ba6b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -19,8 +19,8 @@ coverage.xml .pytest_cache .hypothesis -module/tests -module/conf/const_dev.py +src/module/tests +src/module/conf/const_dev.py config/bangumi.json/config/bangumi.json /docs /.github diff --git a/.github/workflows/dev-latest.yml b/.github/workflows/dev-latest.yml index 2071149a..da55a6dc 100644 --- a/.github/workflows/dev-latest.yml +++ b/.github/workflows/dev-latest.yml @@ -12,7 +12,7 @@ jobs: uses: actions/checkout@v3 - name: Create Version info run: | - echo "VERSION = '2.6.0-beta'" > module/conf/version.py + echo "VERSION = '2.6.0-beta'" > module/__version__.py - name: Set up QEMU uses: docker/setup-qemu-action@v2 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d25d4aad..caa3c847 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -14,7 +14,7 @@ jobs: uses: actions/checkout@v3 - name: Create Version info run: | - echo "version='$GITHUB_REF_NAME'" > module/conf/version.py + echo "version='$GITHUB_REF_NAME'" > src/module/__version__.py - name: Docker meta id: meta uses: docker/metadata-action@v4 diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 94a59282..03b0584a 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -17,17 +17,18 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@v3 with: - python-version: "3.10" + python-version: "3.11" - name: Install dependencies run: | python -m pip install --upgrade pip if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip install pytest - mkdir config - name: Test - run: pytest test/test_raw_parser.py \ No newline at end of file + run: | + cd src + pytest test/test_raw_parser.py \ No newline at end of file diff --git a/.gitignore b/.gitignore index d2eda412..770d221f 100644 --- a/.gitignore +++ b/.gitignore @@ -162,21 +162,19 @@ cython_debug/ #.idea/ # Custom -/module/conf/const_dev.py -/config -/module/tester.py -/module/config +/src/test.py -/module/parser/analyser/tmdb_parser.py +/src/module/parser/analyser/tmdb_parser.py -/module/run_debug.sh -/module/debug_run.sh -/module/__version__.py -/data/ +/src/module/run_debug.sh +/src/module/debug_run.sh +/src/module/__version__.py +/src/data/ -/module/conf/config_dev.ini +/src/module/conf/config_dev.ini test.* .run -/module/conf/version.py -/templates/ + +/src/templates/ +/src/config/ diff --git a/Dockerfile b/Dockerfile index 9035d817..cc6fddc1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,7 @@ ENV TZ=Asia/Shanghai \ WORKDIR /app COPY --from=build --chmod=777 /install /usr/local -COPY --chmod=755 . /app +COPY --chmod=755 src/. /app RUN apk add --no-cache \ curl \ diff --git a/main.py b/main.py deleted file mode 100644 index 29fffaab..00000000 --- a/main.py +++ /dev/null @@ -1,16 +0,0 @@ -from module import app -from module import api - -import multiprocessing - -if __name__ == "__main__": - num_processes = 2 - processes = [] - p1 = multiprocessing.Process(target=app.run) - p2 = multiprocessing.Process(target=api.run) - process_list = [p1, p2] - for p in process_list: - p.start() - processes.append(p) - for p in processes: - p.join() \ No newline at end of file diff --git a/module/core/__init__.py b/module/core/__init__.py deleted file mode 100644 index 4ba677f3..00000000 --- a/module/core/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .download_client import DownloadClient -from .eps_complete import FullSeasonGet -from .renamer import Renamer -from .rss_analyser import RSSAnalyser -from .api_func import APIProcess diff --git a/setup.py b/setup.py deleted file mode 100644 index dafa12a7..00000000 --- a/setup.py +++ /dev/null @@ -1,27 +0,0 @@ -# coding:utf-8 - -from setuptools import setup, find_packages - -setup( - name="auto_bangumi", # 包名字 - version="2.4.0b4", # 包版本 - description="一个全自动追番整理下载工具", - long_description=""" - 本项目是基于 Mikan Project、qBittorrent 的全自动追番整理下载工具。 - 只需要在 Mikan Project 上订阅番剧，就可以全自动追番。 - 并且整理完成的名称和目录可以直接被 Plex、Jellyfin 等媒体库软件识别， - 无需二次刮削。""", # 简单描述 - author="EstrellaXD", # 作者 - author_email="estrellaxd05@gmail.com", # 作者邮箱 - url="https://github.com/EstrellaXD/Auto_Bangumi", # 包的主页 - packages=find_packages(where=".", exclude=("tests",), include=('*',)), - package_data={"auto_bangumi.RssFilter": ["*.json"]}, - package_dir={"auto_bangumi": "auto_bangumi"}, - install_requires= [ - "qbittorrent-api", - "bs4", - "requests", - "lxml", - "zhconv", - ] -) diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..1b22df76 --- /dev/null +++ b/src/main.py @@ -0,0 +1,18 @@ +from module import app +from module import api + +import multiprocessing + +if __name__ == "__main__": + # num_processes = 2 + # processes = [] + # p1 = multiprocessing.Process(target=app.run) + # p2 = multiprocessing.Process(target=api.run) + # process_list = [p1, p2] + # for p in process_list: + # p.start() + # processes.append(p) + # for p in processes: + # p.join() + app.run() + diff --git a/module/__init__.py b/src/module/__init__.py similarity index 100% rename from module/__init__.py rename to src/module/__init__.py diff --git a/module/ab_decorator/__init__.py b/src/module/ab_decorator/__init__.py similarity index 100% rename from module/ab_decorator/__init__.py rename to src/module/ab_decorator/__init__.py diff --git a/module/api.py b/src/module/api.py similarity index 100% rename from module/api.py rename to src/module/api.py diff --git a/module/app.py b/src/module/app.py similarity index 92% rename from module/app.py rename to src/module/app.py index 9d10ea1c..6904f384 100644 --- a/module/app.py +++ b/src/module/app.py @@ -5,8 +5,9 @@ import logging from module.conf import settings, setup_logger, LOG_PATH, DATA_PATH, VERSION from module.utils import json_config -from module.core import RSSAnalyser, DownloadClient, Renamer, FullSeasonGet - +from module.core import DownloadClient +from module.manager import Renamer, FullSeasonGet +from module.rss import RSSAnalyser logger = logging.getLogger(__name__) @@ -28,8 +29,8 @@ def load_data_file(): bangumi_data = json_config.load(DATA_PATH) if bangumi_data["data_version"] != settings.program.data_version or bangumi_data["rss_link"] != settings.rss_parser.link: bangumi_data = { - "rss_link": settings.rss_link, - "data_version": settings.data_version, + "rss_link": settings.rss_parser.link, + "data_version": settings.program.data_version, "bangumi_info": [] } logger.info("Rebuilding data information...") diff --git a/module/conf/__init__.py b/src/module/conf/__init__.py similarity index 100% rename from module/conf/__init__.py rename to src/module/conf/__init__.py index c67557ea..a3b86de5 100644 --- a/module/conf/__init__.py +++ b/src/module/conf/__init__.py @@ -1,5 +1,5 @@ -from .config import settings, VERSION from .log import setup_logger, LOG_PATH +from .config import settings, VERSION TMDB_API = "32b19d6a05b512190a056fa4e747cbbc" diff --git a/module/conf/config.py b/src/module/conf/config.py similarity index 86% rename from module/conf/config.py rename to src/module/conf/config.py index 56a88c43..9090f1c4 100644 --- a/module/conf/config.py +++ b/src/module/conf/config.py @@ -5,6 +5,10 @@ from dataclasses import dataclass from .const import DEFAULT_SETTINGS, ENV_TO_ATTR +try: + from ..__version__ import VERSION +except ImportError: + VERSION = "DEV_VERSION" class ConfLoad(dict): @@ -29,7 +33,7 @@ class Settings: self.load(path) def load(self, path: str | None): - if isinstance(path, dict): + if path is None: conf = DEFAULT_SETTINGS elif os.path.isfile(path): with open(path, "r") as f: @@ -58,15 +62,13 @@ class Settings: json.dump(settings, f, indent=4) return settings -try: - from .version import VERSION - if os.path.isdir("config"): - CONFIG_PATH = "config/config.json" - else: - CONFIG_PATH = None -except ImportError: - VERSION = "DEV_VERSION" + +if os.path.isdir("config") and VERSION == "DEV_VERSION": CONFIG_PATH = "config/config_dev.json" +elif os.path.isdir("config") and VERSION != "DEV_VERSION": + CONFIG_PATH = "config/config.json" +else: + CONFIG_PATH = None settings = Settings(CONFIG_PATH) diff --git a/module/conf/const.py b/src/module/conf/const.py similarity index 100% rename from module/conf/const.py rename to src/module/conf/const.py diff --git a/module/conf/log.py b/src/module/conf/log.py similarity index 94% rename from module/conf/log.py rename to src/module/conf/log.py index c110a67d..b8516603 100644 --- a/module/conf/log.py +++ b/src/module/conf/log.py @@ -1,5 +1,6 @@ import logging -from module.conf import settings + +from .config import settings LOG_PATH = "data/log.txt" diff --git a/module/conf/logging.conf b/src/module/conf/logging.conf similarity index 100% rename from module/conf/logging.conf rename to src/module/conf/logging.conf diff --git a/module/conf/parse.py b/src/module/conf/parse.py similarity index 100% rename from module/conf/parse.py rename to src/module/conf/parse.py diff --git a/src/module/core/__init__.py b/src/module/core/__init__.py new file mode 100644 index 00000000..596ca235 --- /dev/null +++ b/src/module/core/__init__.py @@ -0,0 +1,2 @@ +from .download_client import DownloadClient +from .api_func import APIProcess diff --git a/module/core/api_func.py b/src/module/core/api_func.py similarity index 94% rename from module/core/api_func.py rename to src/module/core/api_func.py index 2c137014..9c48d242 100644 --- a/module/core/api_func.py +++ b/src/module/core/api_func.py @@ -1,6 +1,8 @@ import re -from module.core import FullSeasonGet, DownloadClient, RSSAnalyser +from module.core import DownloadClient +from module.manager import FullSeasonGet +from module.rss import RSSAnalyser from module.utils import json_config from module.conf import DATA_PATH diff --git a/module/core/download_client.py b/src/module/core/download_client.py similarity index 96% rename from module/core/download_client.py rename to src/module/core/download_client.py index 9fa21879..6d92b47d 100644 --- a/module/core/download_client.py +++ b/src/module/core/download_client.py @@ -3,7 +3,6 @@ import logging import os from module.downloader import getClient -from module.downloader.exceptions import ConflictError from module.conf import settings @@ -115,8 +114,3 @@ class DownloadClient: def get_torrent_path(self, hashes): return self.client.get_torrent_path(hashes) - -if __name__ == "__main__": - put = DownloadClient() - put.rss_feed() - diff --git a/module/core/download_fliter.py b/src/module/core/download_fliter.py similarity index 100% rename from module/core/download_fliter.py rename to src/module/core/download_fliter.py diff --git a/module/downloader/__init__.py b/src/module/downloader/__init__.py similarity index 71% rename from module/downloader/__init__.py rename to src/module/downloader/__init__.py index 08bbffbc..2dd7a157 100644 --- a/module/downloader/__init__.py +++ b/src/module/downloader/__init__.py @@ -7,5 +7,5 @@ def getClient(): password = settings.downloader.password # TODO 多下载器支持 # 从 settings 里读取下载器名称，然后返回对应 Client - from module.downloader.qb_downloader import QbDownloader - return QbDownloader(host, username, password) \ No newline at end of file + from .qb_downloader import QbDownloader + return QbDownloader(host, username, password) diff --git a/module/downloader/aria2_downloader.py b/src/module/downloader/aria2_downloader.py similarity index 100% rename from module/downloader/aria2_downloader.py rename to src/module/downloader/aria2_downloader.py diff --git a/module/downloader/exceptions.py b/src/module/downloader/exceptions.py similarity index 100% rename from module/downloader/exceptions.py rename to src/module/downloader/exceptions.py diff --git a/module/downloader/qb_downloader.py b/src/module/downloader/qb_downloader.py similarity index 100% rename from module/downloader/qb_downloader.py rename to src/module/downloader/qb_downloader.py diff --git a/module/downloader/tr_downloader.py b/src/module/downloader/tr_downloader.py similarity index 100% rename from module/downloader/tr_downloader.py rename to src/module/downloader/tr_downloader.py diff --git a/src/module/manager/__init__.py b/src/module/manager/__init__.py new file mode 100644 index 00000000..410f242f --- /dev/null +++ b/src/module/manager/__init__.py @@ -0,0 +1,2 @@ +from .eps_complete import FullSeasonGet +from .renamer import Renamer \ No newline at end of file diff --git a/module/core/eps_complete.py b/src/module/manager/eps_complete.py similarity index 98% rename from module/core/eps_complete.py rename to src/module/manager/eps_complete.py index 87c2a3c9..b6ce13cf 100644 --- a/module/core/eps_complete.py +++ b/src/module/manager/eps_complete.py @@ -5,7 +5,7 @@ import logging from module.conf import settings from module.network import RequestContent -from .download_client import DownloadClient +from module.core.download_client import DownloadClient logger = logging.getLogger(__name__) SEARCH_KEY = ["group", "title_raw", "season_raw", "subtitle", "source", "dpi"] diff --git a/module/core/renamer.py b/src/module/manager/renamer.py similarity index 90% rename from module/core/renamer.py rename to src/module/manager/renamer.py index 7cd681f2..8a5d44cf 100644 --- a/module/core/renamer.py +++ b/src/module/manager/renamer.py @@ -1,14 +1,13 @@ import logging import os.path import re -import os.path from pathlib import PurePath, PureWindowsPath -from .download_client import DownloadClient +from module.core.download_client import DownloadClient from module.conf import settings from module.parser import TitleParser -from ..network import PostNotification, ServerChanNotification +from module.network import PostNotification, ServerChanNotification logger = logging.getLogger(__name__) @@ -54,7 +53,6 @@ class Renamer: return path_name, season, folder_name, suffix, download_path def run(self): - notification = ServerChanNotification() recent_info, torrent_count = self.get_torrent_info() rename_count = 0 for info in recent_info: @@ -67,10 +65,9 @@ class Renamer: try: new_name = self._renamer.download_parser(name, folder_name, season, suffix, settings.bangumi_manage.rename_method) if path_name != new_name: - old_name = info.content_path.replace(info.save_path, "") + old_name = os.path.basename(info.content_path) self.client.rename_torrent_file(torrent_hash, new_name, old_name, new_name) rename_count += 1 - notification.send_msg(f"《{name[:10]}》缓存成功", f"[Auto Bangumi]《{name}》缓存成功") else: continue except Exception as e: diff --git a/module/core/repath.py b/src/module/manager/repath.py similarity index 100% rename from module/core/repath.py rename to src/module/manager/repath.py diff --git a/module/models/__init__.py b/src/module/models/__init__.py similarity index 100% rename from module/models/__init__.py rename to src/module/models/__init__.py diff --git a/module/models/api.py b/src/module/models/api.py similarity index 100% rename from module/models/api.py rename to src/module/models/api.py diff --git a/module/models/bangumi.py b/src/module/models/bangumi.py similarity index 100% rename from module/models/bangumi.py rename to src/module/models/bangumi.py diff --git a/module/network/__init__.py b/src/module/network/__init__.py similarity index 100% rename from module/network/__init__.py rename to src/module/network/__init__.py diff --git a/module/network/notification.py b/src/module/network/notification.py similarity index 76% rename from module/network/notification.py rename to src/module/network/notification.py index 834f9a68..50860b90 100644 --- a/module/network/notification.py +++ b/src/module/network/notification.py @@ -19,6 +19,20 @@ class PostNotification: return response.status_code == 200 +class TelegramNotification: + def __init__(self): + self.token = settings.notification_token + self.notification_url = f"https://api.telegram.org/bot{self.token}/sendMessage" + + def send_msg(self, title: str, desp: str) -> bool: + if not settings.notification_enable: + return False + data = { + "chat_id": settings.notification_chat_id, + "text": f"{title}\n{desp}", + } + + class ServerChanNotification: """Server酱推送""" diff --git a/module/network/request_contents.py b/src/module/network/request_contents.py similarity index 99% rename from module/network/request_contents.py rename to src/module/network/request_contents.py index 6a0bfbf6..497c89c1 100644 --- a/module/network/request_contents.py +++ b/src/module/network/request_contents.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from bs4 import BeautifulSoup from .request_url import RequestURL + from module.conf import settings import re diff --git a/module/network/request_url.py b/src/module/network/request_url.py similarity index 100% rename from module/network/request_url.py rename to src/module/network/request_url.py diff --git a/module/parser/__init__.py b/src/module/parser/__init__.py similarity index 100% rename from module/parser/__init__.py rename to src/module/parser/__init__.py diff --git a/module/parser/analyser/__init__.py b/src/module/parser/analyser/__init__.py similarity index 100% rename from module/parser/analyser/__init__.py rename to src/module/parser/analyser/__init__.py diff --git a/module/parser/analyser/bgm_parser.py b/src/module/parser/analyser/bgm_parser.py similarity index 100% rename from module/parser/analyser/bgm_parser.py rename to src/module/parser/analyser/bgm_parser.py diff --git a/module/parser/analyser/raw_parser.py b/src/module/parser/analyser/raw_parser.py similarity index 100% rename from module/parser/analyser/raw_parser.py rename to src/module/parser/analyser/raw_parser.py diff --git a/module/parser/analyser/rename_parser.py b/src/module/parser/analyser/rename_parser.py similarity index 100% rename from module/parser/analyser/rename_parser.py rename to src/module/parser/analyser/rename_parser.py diff --git a/module/parser/analyser/tmdb_parser.py b/src/module/parser/analyser/tmdb_parser.py similarity index 100% rename from module/parser/analyser/tmdb_parser.py rename to src/module/parser/analyser/tmdb_parser.py diff --git a/module/parser/fuzz_match.py b/src/module/parser/fuzz_match.py similarity index 100% rename from module/parser/fuzz_match.py rename to src/module/parser/fuzz_match.py diff --git a/module/parser/title_parser.py b/src/module/parser/title_parser.py similarity index 100% rename from module/parser/title_parser.py rename to src/module/parser/title_parser.py diff --git a/src/module/rss/__init__.py b/src/module/rss/__init__.py new file mode 100644 index 00000000..9650cb6c --- /dev/null +++ b/src/module/rss/__init__.py @@ -0,0 +1 @@ +from .rss_analyser import RSSAnalyser \ No newline at end of file diff --git a/module/core/rss_analyser.py b/src/module/rss/rss_analyser.py similarity index 99% rename from module/core/rss_analyser.py rename to src/module/rss/rss_analyser.py index aa611e30..9060e27b 100644 --- a/module/core/rss_analyser.py +++ b/src/module/rss/rss_analyser.py @@ -1,6 +1,5 @@ import re import logging - from module.network import RequestContent from module.parser import TitleParser diff --git a/module/setID.sh b/src/module/setID.sh similarity index 100% rename from module/setID.sh rename to src/module/setID.sh diff --git a/module/utils/__init__.py b/src/module/utils/__init__.py similarity index 100% rename from module/utils/__init__.py rename to src/module/utils/__init__.py diff --git a/module/utils/json_config.py b/src/module/utils/json_config.py similarity index 100% rename from module/utils/json_config.py rename to src/module/utils/json_config.py diff --git a/run.sh b/src/run.sh similarity index 100% rename from run.sh rename to src/run.sh diff --git a/test/__init__.py b/src/test/__init__.py similarity index 100% rename from test/__init__.py rename to src/test/__init__.py diff --git a/test/test_raw_parser.py b/src/test/test_raw_parser.py similarity index 100% rename from test/test_raw_parser.py rename to src/test/test_raw_parser.py From cc65c07886b44c3ef471b09e13ee217e240b14b8 Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Tue, 14 Mar 2023 23:11:45 +0800 Subject: [PATCH 15/30] Fix RequestContent(), fix renamer.py --- .github/workflows/dev-latest.yml | 3 +++ .github/workflows/docker.yml | 5 ++++- .github/workflows/unittest.yml | 5 ++++- .gitignore | 1 + .dockerignore => src/.dockerignore | 2 +- Dockerfile => src/Dockerfile | 4 ++-- src/main.py | 21 ++++++++++----------- src/module/api.py | 2 +- src/module/manager/eps_complete.py | 15 --------------- src/module/manager/renamer.py | 5 +++-- src/module/models/__init__.py | 1 + src/module/network/request_url.py | 1 + requirements.txt => src/requirements.txt | 0 src/run.sh | 1 - 14 files changed, 31 insertions(+), 35 deletions(-) rename .dockerignore => src/.dockerignore (95%) rename Dockerfile => src/Dockerfile (94%) rename requirements.txt => src/requirements.txt (100%) diff --git a/.github/workflows/dev-latest.yml b/.github/workflows/dev-latest.yml index da55a6dc..966c0a79 100644 --- a/.github/workflows/dev-latest.yml +++ b/.github/workflows/dev-latest.yml @@ -8,6 +8,9 @@ jobs: latest: runs-on: ubuntu-latest steps: + - name: Enter directory + run: | + cd src - name: Checkout uses: actions/checkout@v3 - name: Create Version info diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index caa3c847..4639854c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -9,12 +9,15 @@ jobs: docker: runs-on: ubuntu-latest steps: + - name: Enter directory + run: | + cd src - name: Checkout uses: actions/checkout@v3 - name: Create Version info run: | - echo "version='$GITHUB_REF_NAME'" > src/module/__version__.py + echo "VERSION = '$GITHUB_REF_NAME'" > module/__version__.py - name: Docker meta id: meta uses: docker/metadata-action@v4 diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 03b0584a..b4448a7f 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -22,6 +22,10 @@ jobs: with: python-version: "3.11" + - name: Enter directory + run: | + cd src + - name: Install dependencies run: | python -m pip install --upgrade pip @@ -30,5 +34,4 @@ jobs: - name: Test run: | - cd src pytest test/test_raw_parser.py \ No newline at end of file diff --git a/.gitignore b/.gitignore index 770d221f..b161e24b 100644 --- a/.gitignore +++ b/.gitignore @@ -178,3 +178,4 @@ test.* /src/templates/ /src/config/ +/src/debuger.py diff --git a/.dockerignore b/src/.dockerignore similarity index 95% rename from .dockerignore rename to src/.dockerignore index e597ba6b..6713638d 100644 --- a/.dockerignore +++ b/src/.dockerignore @@ -16,7 +16,7 @@ coverage.xml *.log .git .mypy_cache -.pytest_cache +../.pytest_cache .hypothesis src/module/tests diff --git a/Dockerfile b/src/Dockerfile similarity index 94% rename from Dockerfile rename to src/Dockerfile index cc6fddc1..1c4c7b75 100644 --- a/Dockerfile +++ b/src/Dockerfile @@ -25,7 +25,7 @@ ENV TZ=Asia/Shanghai \ WORKDIR /app COPY --from=build --chmod=777 /install /usr/local -COPY --chmod=755 src/. /app +COPY --chmod=755 . /app RUN apk add --no-cache \ curl \ @@ -48,6 +48,6 @@ RUN addgroup -S auto_bangumi -g 1000 && \ EXPOSE 7892 -VOLUME [ "config" , "data"] +VOLUME [ "/app/config" , "/app/data"] CMD ["sh", "run.sh"] diff --git a/src/main.py b/src/main.py index 1b22df76..4496dec6 100644 --- a/src/main.py +++ b/src/main.py @@ -4,15 +4,14 @@ from module import api import multiprocessing if __name__ == "__main__": - # num_processes = 2 - # processes = [] - # p1 = multiprocessing.Process(target=app.run) - # p2 = multiprocessing.Process(target=api.run) - # process_list = [p1, p2] - # for p in process_list: - # p.start() - # processes.append(p) - # for p in processes: - # p.join() - app.run() + num_processes = 2 + processes = [] + p1 = multiprocessing.Process(target=app.run) + p2 = multiprocessing.Process(target=api.run) + process_list = [p1, p2] + for p in process_list: + p.start() + processes.append(p) + for p in processes: + p.join() diff --git a/src/module/api.py b/src/module/api.py index 1d6fdb83..eb789a93 100644 --- a/src/module/api.py +++ b/src/module/api.py @@ -9,7 +9,7 @@ import logging from .core import APIProcess from .conf import settings, DATA_PATH, LOG_PATH from .utils import json_config -from .models.api import * +from models.api import * logger = logging.getLogger(__name__) diff --git a/src/module/manager/eps_complete.py b/src/module/manager/eps_complete.py index b6ce13cf..ee404629 100644 --- a/src/module/manager/eps_complete.py +++ b/src/module/manager/eps_complete.py @@ -66,18 +66,3 @@ class FullSeasonGet: logger.info("Completed!") -if __name__ == "__main__": - a = FullSeasonGet() - data = { - "official_title": "指名！", - "title_raw": "CUE!", - "season": 1, - "season_raw": "", - "group": "喵萌Production", - "dpi": "1080p", - "source": None, - "subtitle": "简日双语", - "added": True, - "eps_collect": True - } - print(a.init_eps_complete_search_str(data)) diff --git a/src/module/manager/renamer.py b/src/module/manager/renamer.py index 8a5d44cf..0782377c 100644 --- a/src/module/manager/renamer.py +++ b/src/module/manager/renamer.py @@ -65,8 +65,9 @@ class Renamer: try: new_name = self._renamer.download_parser(name, folder_name, season, suffix, settings.bangumi_manage.rename_method) if path_name != new_name: - old_name = os.path.basename(info.content_path) - self.client.rename_torrent_file(torrent_hash, new_name, old_name, new_name) + old_path = info.content_path.replace(info.save_path, "") + old_path = old_path[len(os.path.sep):] + self.client.rename_torrent_file(torrent_hash, new_name, old_path, new_name) rename_count += 1 else: continue diff --git a/src/module/models/__init__.py b/src/module/models/__init__.py index e69de29b..8e0e9d66 100644 --- a/src/module/models/__init__.py +++ b/src/module/models/__init__.py @@ -0,0 +1 @@ +from .bangumi import * \ No newline at end of file diff --git a/src/module/network/request_url.py b/src/module/network/request_url.py index 5dbd30e6..90be4c1a 100644 --- a/src/module/network/request_url.py +++ b/src/module/network/request_url.py @@ -48,6 +48,7 @@ class RequestURL: socks.set_default_proxy(socks.SOCKS5, addr=settings.proxy.host, port=settings.proxy.port, rdns=True, username=settings.proxy.username, password=settings.proxy.password) socket.socket = socks.socksocket + return self def __exit__(self, exc_type, exc_val, exc_tb): self.session.close() diff --git a/requirements.txt b/src/requirements.txt similarity index 100% rename from requirements.txt rename to src/requirements.txt diff --git a/src/run.sh b/src/run.sh index 1a8e98ee..39000cb9 100755 --- a/src/run.sh +++ b/src/run.sh @@ -6,6 +6,5 @@ if [ -f /config/bangumi.json ]; then fi - umask ${UMASK} python3 main.py From c3b0e4c086e86a48f7e281053e0dfe5268faf569 Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Tue, 14 Mar 2023 23:18:23 +0800 Subject: [PATCH 16/30] Fix pytest --- .github/workflows/unittest.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index b4448a7f..c8380698 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -25,13 +25,11 @@ jobs: - name: Enter directory run: | cd src - - name: Install dependencies run: | python -m pip install --upgrade pip if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip install pytest - - name: Test run: | - pytest test/test_raw_parser.py \ No newline at end of file + pytest ./test/test_raw_parser.py \ No newline at end of file From df9ebc02e40f5c6e0e1d3afee739732140a7997a Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Tue, 14 Mar 2023 23:18:23 +0800 Subject: [PATCH 17/30] Fix pytest --- .github/workflows/unittest.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index c8380698..d5c6dc46 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -21,15 +21,13 @@ jobs: uses: actions/setup-python@v3 with: python-version: "3.11" - - - name: Enter directory - run: | - cd src - name: Install dependencies + working-directory: ./src run: | python -m pip install --upgrade pip if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip install pytest - name: Test + working-directory: ./src run: | - pytest ./test/test_raw_parser.py \ No newline at end of file + pytest test/test_raw_parser.py \ No newline at end of file From 405458aba54077007c948a3dabadcaf17a1a6b97 Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Tue, 14 Mar 2023 23:23:14 +0800 Subject: [PATCH 18/30] Fix github action --- .github/workflows/dev-latest.yml | 5 ++--- .github/workflows/docker.yml | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/dev-latest.yml b/.github/workflows/dev-latest.yml index 966c0a79..fd644a94 100644 --- a/.github/workflows/dev-latest.yml +++ b/.github/workflows/dev-latest.yml @@ -8,12 +8,10 @@ jobs: latest: runs-on: ubuntu-latest steps: - - name: Enter directory - run: | - cd src - name: Checkout uses: actions/checkout@v3 - name: Create Version info + working-directory: ./src run: | echo "VERSION = '2.6.0-beta'" > module/__version__.py - @@ -30,6 +28,7 @@ jobs: password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - name: Build and push + working-directory: ./src uses: docker/build-push-action@v3 with: push: true diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4639854c..5697b94e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -9,13 +9,11 @@ jobs: docker: runs-on: ubuntu-latest steps: - - name: Enter directory - run: | - cd src - name: Checkout uses: actions/checkout@v3 - name: Create Version info + working-directory: ./src run: | echo "VERSION = '$GITHUB_REF_NAME'" > module/__version__.py - name: Docker meta @@ -39,6 +37,7 @@ jobs: password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - name: Build and push + working-directory: ./src uses: docker/build-push-action@v3 with: context: . From 8b31256e7817dc6692a8564a08a613583d2fcf9d Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Tue, 14 Mar 2023 23:26:20 +0800 Subject: [PATCH 19/30] Fix GitHub action --- src/Dockerfile | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/src/Dockerfile b/src/Dockerfile index 1c4c7b75..69e54d00 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -1,20 +1,4 @@ # syntax=docker/dockerfile:1 -FROM python:3.11-alpine AS build - -RUN mkdir /install -WORKDIR /install -COPY requirements.txt . - -RUN apk update \ - && apk add --no-cache \ - curl \ - gcc \ - g++ \ - linux-headers \ - -RUN python3 -m pip install --upgrade pip \ - && pip install -r requirements.txt --prefix="/install" - FROM python:3.11-alpine ENV TZ=Asia/Shanghai \ @@ -24,7 +8,11 @@ ENV TZ=Asia/Shanghai \ WORKDIR /app -COPY --from=build --chmod=777 /install /usr/local +COPY requirements.txt . + +RUN python3 -m pip install --upgrade pip \ + && pip install -r requirements.txt --no-cache-dir + COPY --chmod=755 . /app RUN apk add --no-cache \ From 4cbfe5dfdf6c59da010cbc41ae61b5cae133f578 Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Tue, 14 Mar 2023 23:28:12 +0800 Subject: [PATCH 20/30] Fix GitHub action of docker.yml --- .github/workflows/dev-latest.yml | 3 +-- .github/workflows/docker.yml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dev-latest.yml b/.github/workflows/dev-latest.yml index fd644a94..5b874e86 100644 --- a/.github/workflows/dev-latest.yml +++ b/.github/workflows/dev-latest.yml @@ -28,9 +28,8 @@ jobs: password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - name: Build and push - working-directory: ./src uses: docker/build-push-action@v3 with: push: true tags: estrellaxd/auto_bangumi:2.6.0-beta2 - file: ./Dockerfile + file: .src/Dockerfile diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5697b94e..8168738a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -37,10 +37,9 @@ jobs: password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - name: Build and push - working-directory: ./src uses: docker/build-push-action@v3 with: - context: . + context: src/. platforms: linux/amd64,linux/arm64,linux/arm push: true tags: ${{ steps.meta.outputs.tags }} From 405e054316aaee9d8873838fe686d80be119fb39 Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Tue, 14 Mar 2023 23:30:42 +0800 Subject: [PATCH 21/30] Fix GitHub action of docker.yml --- .github/workflows/dev-latest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev-latest.yml b/.github/workflows/dev-latest.yml index 5b874e86..61cc2393 100644 --- a/.github/workflows/dev-latest.yml +++ b/.github/workflows/dev-latest.yml @@ -32,4 +32,4 @@ jobs: with: push: true tags: estrellaxd/auto_bangumi:2.6.0-beta2 - file: .src/Dockerfile + file: src/Dockerfile From 01fcac25763a5e4ceee5b70ef41a71f0d5ad284f Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Tue, 14 Mar 2023 23:32:59 +0800 Subject: [PATCH 22/30] Fix GitHub action of docker.yml --- .github/workflows/dev-latest.yml | 3 ++- .github/workflows/docker.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev-latest.yml b/.github/workflows/dev-latest.yml index 61cc2393..fd32249f 100644 --- a/.github/workflows/dev-latest.yml +++ b/.github/workflows/dev-latest.yml @@ -30,6 +30,7 @@ jobs: name: Build and push uses: docker/build-push-action@v3 with: + working-directory: ./src push: true tags: estrellaxd/auto_bangumi:2.6.0-beta2 - file: src/Dockerfile + file: Dockerfile diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8168738a..6263a34e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -39,7 +39,8 @@ jobs: name: Build and push uses: docker/build-push-action@v3 with: - context: src/. + working-directory: ./src + context: . platforms: linux/amd64,linux/arm64,linux/arm push: true tags: ${{ steps.meta.outputs.tags }} From 85bd178b5571608c1c78c4d274a07b1c4e29129b Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Tue, 14 Mar 2023 23:38:28 +0800 Subject: [PATCH 23/30] Fix GitHub action of docker.yml --- .github/workflows/dev-latest.yml | 2 +- .github/workflows/docker.yml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dev-latest.yml b/.github/workflows/dev-latest.yml index fd32249f..dd4bb907 100644 --- a/.github/workflows/dev-latest.yml +++ b/.github/workflows/dev-latest.yml @@ -30,7 +30,7 @@ jobs: name: Build and push uses: docker/build-push-action@v3 with: - working-directory: ./src + context: ./src push: true tags: estrellaxd/auto_bangumi:2.6.0-beta2 file: Dockerfile diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6263a34e..d10311f4 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -39,8 +39,7 @@ jobs: name: Build and push uses: docker/build-push-action@v3 with: - working-directory: ./src - context: . + context: ./src platforms: linux/amd64,linux/arm64,linux/arm push: true tags: ${{ steps.meta.outputs.tags }} From 4a0cd5bae034564404837d036144b35341855ebd Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Tue, 14 Mar 2023 23:40:15 +0800 Subject: [PATCH 24/30] Fix GitHub action of docker.yml --- .github/workflows/dev-latest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev-latest.yml b/.github/workflows/dev-latest.yml index dd4bb907..fa97c9d9 100644 --- a/.github/workflows/dev-latest.yml +++ b/.github/workflows/dev-latest.yml @@ -30,7 +30,7 @@ jobs: name: Build and push uses: docker/build-push-action@v3 with: - context: ./src + context: ./src/. push: true tags: estrellaxd/auto_bangumi:2.6.0-beta2 file: Dockerfile From 6ac7de272e8e17c72fa28306518e93e716b24c5a Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Tue, 14 Mar 2023 23:43:16 +0800 Subject: [PATCH 25/30] Fix GitHub action of docker.yml --- src/.dockerignore => .dockerignore | 0 .github/workflows/dev-latest.yml | 2 +- .github/workflows/docker.yml | 2 +- src/Dockerfile => Dockerfile | 4 ++-- 4 files changed, 4 insertions(+), 4 deletions(-) rename src/.dockerignore => .dockerignore (100%) rename src/Dockerfile => Dockerfile (94%) diff --git a/src/.dockerignore b/.dockerignore similarity index 100% rename from src/.dockerignore rename to .dockerignore diff --git a/.github/workflows/dev-latest.yml b/.github/workflows/dev-latest.yml index fa97c9d9..a1193ada 100644 --- a/.github/workflows/dev-latest.yml +++ b/.github/workflows/dev-latest.yml @@ -30,7 +30,7 @@ jobs: name: Build and push uses: docker/build-push-action@v3 with: - context: ./src/. + context: . push: true tags: estrellaxd/auto_bangumi:2.6.0-beta2 file: Dockerfile diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d10311f4..c9c567a1 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -39,7 +39,7 @@ jobs: name: Build and push uses: docker/build-push-action@v3 with: - context: ./src + context: . platforms: linux/amd64,linux/arm64,linux/arm push: true tags: ${{ steps.meta.outputs.tags }} diff --git a/src/Dockerfile b/Dockerfile similarity index 94% rename from src/Dockerfile rename to Dockerfile index 69e54d00..d7b99cbe 100644 --- a/src/Dockerfile +++ b/Dockerfile @@ -8,12 +8,12 @@ ENV TZ=Asia/Shanghai \ WORKDIR /app -COPY requirements.txt . +COPY src/requirements.txt . RUN python3 -m pip install --upgrade pip \ && pip install -r requirements.txt --no-cache-dir -COPY --chmod=755 . /app +COPY --chmod=755 src/. /app/. RUN apk add --no-cache \ curl \ From 93865e4ef73f784633a16602d6b8f519092d1a75 Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Tue, 14 Mar 2023 23:54:14 +0800 Subject: [PATCH 26/30] Fix models-bug --- src/module/api.py | 2 +- src/module/models/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/module/api.py b/src/module/api.py index eb789a93..1d6fdb83 100644 --- a/src/module/api.py +++ b/src/module/api.py @@ -9,7 +9,7 @@ import logging from .core import APIProcess from .conf import settings, DATA_PATH, LOG_PATH from .utils import json_config -from models.api import * +from .models.api import * logger = logging.getLogger(__name__) diff --git a/src/module/models/__init__.py b/src/module/models/__init__.py index 8e0e9d66..1cf896d6 100644 --- a/src/module/models/__init__.py +++ b/src/module/models/__init__.py @@ -1 +1 @@ -from .bangumi import * \ No newline at end of file +from .bangumi import * From 278d23f3b9e7380055ed0ddf8c03e5bfa83878e4 Mon Sep 17 00:00:00 2001 From: Bluefissure Date: Tue, 14 Mar 2023 18:35:58 +0000 Subject: [PATCH 27/30] chore: unnecessary lambda --- src/module/conf/const.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/module/conf/const.py b/src/module/conf/const.py index 0a8fae7e..d81a6bd7 100644 --- a/src/module/conf/const.py +++ b/src/module/conf/const.py @@ -55,9 +55,9 @@ DEFAULT_SETTINGS = { ENV_TO_ATTR = { "program": { - "AB_INTERVAL_TIME": ("sleep_time", lambda e: float(e)), - "AB_RENAME_FREQ": ("times", lambda e: float(e)), - "AB_WEBUI_PORT": ("webui_port", lambda e: int(e)), + "AB_INTERVAL_TIME": ("sleep_time", float), + "AB_RENAME_FREQ": ("times", float), + "AB_WEBUI_PORT": ("webui_port", int), }, "downloader": { "AB_DOWNLOADER_HOST": "host", From b9019e2265b7ca46dfc751feb7515a4d6e1c403a Mon Sep 17 00:00:00 2001 From: Bluefissure Date: Tue, 14 Mar 2023 18:36:59 +0000 Subject: [PATCH 28/30] fix: settings has no download_path anymore --- src/module/manager/eps_complete.py | 2 +- src/module/manager/repath.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/module/manager/eps_complete.py b/src/module/manager/eps_complete.py index ee404629..a12cb8b7 100644 --- a/src/module/manager/eps_complete.py +++ b/src/module/manager/eps_complete.py @@ -35,7 +35,7 @@ class FullSeasonGet: download_info = { "url": torrent.torrent_link, "save_path": os.path.join( - settings.download_path, + settings.downloader.path, data["official_title"], f"Season {data['season']}") } diff --git a/src/module/manager/repath.py b/src/module/manager/repath.py index 48bc6b7b..99e32cd3 100644 --- a/src/module/manager/repath.py +++ b/src/module/manager/repath.py @@ -45,7 +45,7 @@ class RePath: path = rules.get(rule).savePath must_contain = rules.get(rule).mustContain season, folder_name = self.analyse_path(path) - new_path = PurePath(settings.download_path, folder_name, f"Season {season}").__str__() + new_path = PurePath(settings.downloader.path, folder_name, f"Season {season}").__str__() all_rule.append(RuleInfo(rule, must_contain, season, folder_name, new_path)) return all_rule From cdac58dd141c131521439a7d87e7f76cc1d2a98e Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Wed, 15 Mar 2023 10:47:25 +0800 Subject: [PATCH 29/30] Update github action --- .github/workflows/dev-latest.yml | 10 +++++++--- .github/workflows/docker.yml | 22 +++++++++++----------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/.github/workflows/dev-latest.yml b/.github/workflows/dev-latest.yml index a1193ada..db6ce2e1 100644 --- a/.github/workflows/dev-latest.yml +++ b/.github/workflows/dev-latest.yml @@ -1,8 +1,9 @@ name: Deploy To Dockerhub(dev) on: - workflow_dispatch: - branches: [ 2.6.0-dev ] + push: + tags: + - '\d+\.\d+\.\d+-beta\d+' jobs: latest: @@ -32,5 +33,8 @@ jobs: with: context: . push: true - tags: estrellaxd/auto_bangumi:2.6.0-beta2 + tags: | + estrellaxd/auto_bangumi:dev-latest + estrellaxd/auto_bangumi:dev-${{ github.sha }} + estrellaxd/auto_bangumi:${{ github.ref }} file: Dockerfile diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c9c567a1..3c13a29e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,9 +1,13 @@ name: Build(Docker) on: - release: - types: - - released + push: + tags: + - '\d+\.\d+\.\d+' + pull_request: + tags: + - '\d+\.\d+\.\d+' + jobs: docker: @@ -16,13 +20,6 @@ jobs: working-directory: ./src run: | echo "VERSION = '$GITHUB_REF_NAME'" > module/__version__.py - - name: Docker meta - id: meta - uses: docker/metadata-action@v4 - with: - images: ${{ secrets.DOCKER_HUB_USERNAME }}/auto_bangumi - tags: | - type=ref, event=tag - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -42,5 +39,8 @@ jobs: context: . platforms: linux/amd64,linux/arm64,linux/arm push: true - tags: ${{ steps.meta.outputs.tags }} + tags: | + estrellaxd/auto_bangumi:latest + estrellaxd/auto_bangumi:${{ github.sha }} + estrellaxd/auto_bangumi:${{ github.ref }} labels: ${{ steps.meta.outputs.labels }} From 25628d18a51f5a6a4d10d09f18eda8b94ccdae15 Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Wed, 15 Mar 2023 10:50:26 +0800 Subject: [PATCH 30/30] Update models --- src/module/models/api.py | 3 ++- src/module/models/bangumi.py | 14 ++++++++++++++ src/module/parser/analyser/raw_parser.py | 15 ++------------- src/module/parser/fuzz_match.py | 14 ++------------ src/module/parser/title_parser.py | 2 ++ 5 files changed, 22 insertions(+), 26 deletions(-) diff --git a/src/module/models/api.py b/src/module/models/api.py index 9843e7cb..6cad39aa 100644 --- a/src/module/models/api.py +++ b/src/module/models/api.py @@ -15,4 +15,5 @@ class ChangeConfig(BaseModel): class ChangeRule(BaseModel): - rule: dict \ No newline at end of file + rule: dict + diff --git a/src/module/models/bangumi.py b/src/module/models/bangumi.py index c175bc10..cb66fc45 100644 --- a/src/module/models/bangumi.py +++ b/src/module/models/bangumi.py @@ -8,7 +8,21 @@ class MatchRule: rss_link: str +@dataclass class GroupFilter: name: str filter: list + +@dataclass +class Episode: + title_en: str | None + title_zh: str | None + title_jp: str | None + season: int + season_raw: str + episode: int + sub: str + group: str + resolution: str + source: str diff --git a/src/module/parser/analyser/raw_parser.py b/src/module/parser/analyser/raw_parser.py index 72fb7713..e2c093c7 100644 --- a/src/module/parser/analyser/raw_parser.py +++ b/src/module/parser/analyser/raw_parser.py @@ -2,7 +2,7 @@ import logging import re from dataclasses import dataclass -# from parser.episode import Episode +from module.models import Episode logger = logging.getLogger(__name__) @@ -30,18 +30,7 @@ CHINESE_NUMBER_MAP = { } -@dataclass -class Episode: - title_en: str or None - title_zh: str or None - title_jp: str or None - season: int - season_raw: str - episode: int - sub: str - group: str - resolution: str - source: str + class RawParser: diff --git a/src/module/parser/fuzz_match.py b/src/module/parser/fuzz_match.py index 924bc890..15b761d6 100644 --- a/src/module/parser/fuzz_match.py +++ b/src/module/parser/fuzz_match.py @@ -1,7 +1,7 @@ from thefuzz import fuzz import logging -from utils import json_config -from conf import settings +from module.utils import json_config +from module.conf import settings logger = logging.getLogger(__name__) @@ -40,13 +40,3 @@ class FuzzMatch: return max_value, max_info["main"] # logger.debug(max(value)) - -if __name__ == "__main__": - from conf.const_dev import DEV_SETTINGS - settings.init(DEV_SETTINGS) - f = FuzzMatch() - name = "勇者、辞职不干了" - value, title = f.find_max_name(name) - print(f"Raw Name: {name} \n" - f"Match Name: {title} \n" - f"Match Value: {value}") diff --git a/src/module/parser/title_parser.py b/src/module/parser/title_parser.py index e918d135..0cfdaad3 100644 --- a/src/module/parser/title_parser.py +++ b/src/module/parser/title_parser.py @@ -1,11 +1,13 @@ import logging from .analyser import RawParser, DownloadParser, TMDBMatcher + from module.conf import settings logger = logging.getLogger(__name__) LANGUAGE = settings.rss_parser.language + class TitleParser: def __init__(self): self._raw_parser = RawParser()