From 7e45230d1b446f4078517025c5c550d7abd087c0 Mon Sep 17 00:00:00 2001 From: WeijiangChen Date: Sun, 12 Mar 2023 21:51:03 +0800 Subject: [PATCH 1/9] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BA=86=E8=AF=B7?= =?UTF-8?q?=E6=B1=82=E7=B1=BB=EF=BC=8C=E9=81=BF=E5=85=8D=E8=AF=B7=E6=B1=82?= =?UTF-8?q?=E5=BF=98=E8=AE=B0=E5=85=B3=EF=BC=8C=E8=AF=B7=E6=B1=82=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5=E9=87=8D=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- module/core/eps_complete.py | 5 ++- module/core/rss_analyser.py | 9 ++--- module/network/request_url.py | 57 +++++++++++++++------------ module/parser/analyser/bgm_parser.py | 10 ++--- module/parser/analyser/tmdb_parser.py | 38 +++++++++--------- 5 files changed, 63 insertions(+), 56 deletions(-) diff --git a/module/core/eps_complete.py b/module/core/eps_complete.py index 2954d5cf..6b1ede72 100644 --- a/module/core/eps_complete.py +++ b/module/core/eps_complete.py @@ -13,7 +13,7 @@ SEARCH_KEY = ["group", "title_raw", "season_raw", "subtitle", "source", "dpi"] class FullSeasonGet: def __init__(self): - self._get_rss = RequestContent() + pass @staticmethod def init_eps_complete_search_str(data: dict): @@ -24,7 +24,8 @@ class FullSeasonGet: def get_season_torrents(self, data: dict): keyword = self.init_eps_complete_search_str(data) - torrents = self._get_rss.get_torrents(f"https://mikanani.me/RSS/Search?searchstr={keyword}") + with RequestContent() as req: + torrents = req.get_torrents(f"https://mikanani.me/RSS/Search?searchstr={keyword}") return torrents @staticmethod diff --git a/module/core/rss_analyser.py b/module/core/rss_analyser.py index 55a04e64..aa611e30 100644 --- a/module/core/rss_analyser.py +++ b/module/core/rss_analyser.py @@ -14,11 +14,10 @@ logger = logging.getLogger(__name__) class RSSAnalyser: def __init__(self): self._title_analyser = TitleParser() - self._request = RequestContent() def rss_to_datas(self, bangumi_info: list) -> list: - rss_torrents = self._request.get_torrents(settings.rss_parser.link) - self._request.close_session() + with RequestContent() as req: + rss_torrents = req.get_torrents(settings.rss_parser.link) for torrent in rss_torrents: raw_title = torrent.name extra_add = True @@ -35,8 +34,8 @@ class RSSAnalyser: return bangumi_info def rss_to_data(self, url) -> dict: - rss_torrents = self._request.get_torrents(url) - self._request.close_session() + with RequestContent() as req: + rss_torrents = req.get_torrents(url) for torrent in rss_torrents: try: data = self._title_analyser.return_dict(torrent.name) diff --git a/module/network/request_url.py b/module/network/request_url.py index bedc6225..4f38734b 100644 --- a/module/network/request_url.py +++ b/module/network/request_url.py @@ -14,7 +14,37 @@ logger = logging.getLogger(__name__) class RequestURL: def __init__(self): - self.session = requests.session() + self.header = { + "user-agent": "Mozilla/5.0", + "Accept": "application/xml" + } + + def get_url(self, url): + times = 0 + while times < 5: + try: + req = self.session.get(url=url, headers=self.header) + req.raise_for_status() + return req + except requests.RequestException as e: + logger.debug(f"URL: {url}") + logger.debug(e) + logger.warning("ERROR with Connection.Please check DNS/Connection settings") + time.sleep(5) + times += 1 + except Exception as e: + logger.debug(f"URL: {url}") + logger.debug(e) + break + + def get_content(self, url, content="xml"): + if content == "xml": + return BeautifulSoup(self.get_url(url).text, content) + elif content == "json": + return self.get_url(url).json() + + def __enter__(self): + self.session = requests.Session() if settings.proxy.enable: if settings.proxy.type == "http": url = f"http://{settings.proxy.host}:{settings.proxy.port}" @@ -26,31 +56,8 @@ class RequestURL: socks.set_default_proxy(socks.SOCKS5, addr=settings.proxy.host, port=settings.proxy.port, rdns=True, username=settings.proxy.username, password=settings.proxy.password) socket.socket = socks.socksocket - self.header = { - "user-agent": "Mozilla/5.0", - "Accept": "application/xml" - } - def get_url(self, url): - times = 0 - while times < 5: - try: - req = self.session.get(url=url, headers=self.header) - return req - except Exception as e: - logger.debug(f"URL: {url}") - logger.debug(e) - logger.warning("ERROR with Connection.Please check DNS/Connection settings") - time.sleep(5) - times += 1 - - def get_content(self, url, content="xml"): - if content == "xml": - return BeautifulSoup(self.get_url(url).text, content) - elif content == "json": - return self.get_url(url).json() - - def close(self): + def __exit__(self, exc_type, exc_val, exc_tb): self.session.close() diff --git a/module/parser/analyser/bgm_parser.py b/module/parser/analyser/bgm_parser.py index acd7eeb1..427cda11 100644 --- a/module/parser/analyser/bgm_parser.py +++ b/module/parser/analyser/bgm_parser.py @@ -7,11 +7,11 @@ class BgmAPI: f"https://api.bgm.tv/search/subject/{e}?type=2" self.info_url = lambda e: \ f"https://api.bgm.tv/subject/{e}" - self._request = RequestContent() def search(self, title): url = self.search_url(title) - contents = self._request.get_json(url)["list"] - if contents.__len__() == 0: - return None - return contents[0]["name"], contents[0]["name_cn"] \ No newline at end of file + with RequestContent() as req: + contents = req.get_json(url)["list"] + if contents.__len__() == 0: + return None + return contents[0]["name"], contents[0]["name_cn"] diff --git a/module/parser/analyser/tmdb_parser.py b/module/parser/analyser/tmdb_parser.py index 9765ed70..a5759344 100644 --- a/module/parser/analyser/tmdb_parser.py +++ b/module/parser/analyser/tmdb_parser.py @@ -22,14 +22,14 @@ class TMDBMatcher: f"https://api.themoviedb.org/3/search/tv?api_key={TMDB_API}&page=1&query={e}&include_adult=false" self.info_url = lambda e: \ f"https://api.themoviedb.org/3/tv/{e}?api_key={TMDB_API}&language=zh-CN" - self._request = RequestContent() def is_animation(self, tv_id) -> bool: url_info = self.info_url(tv_id) - type_id = self._request.get_json(url_info)["genres"] - for type in type_id: - if type.get("id") == 16: - return True + with RequestContent() as req: + type_id = req.get_json(url_info)["genres"] + for type in type_id: + if type.get("id") == 16: + return True return False # def get_zh_title(self, id): @@ -51,20 +51,20 @@ class TMDBMatcher: return int(re.findall(r"\d", season.get("season"))[0]) def tmdb_search(self, title) -> TMDBInfo: - url = self.search_url(title) - contents = self._request.get_json(url).get("results") - if contents.__len__() == 0: - url = self.search_url(title.replace(" ", "")) - contents = self._request.get_json(url).get("results") - # 判断动画 - for content in contents: - id = content["id"] - if self.is_animation(id): - break - url_info = self.info_url(id) - info_content = self._request.get_json(url_info) - # 关闭链接 - self._request.close() + with RequestContent() as req: + url = self.search_url(title) + contents = req.get_json(url).get("results") + if contents.__len__() == 0: + url = self.search_url(title.replace(" ", "")) + contents = req.get_json(url).get("results") + # 判断动画 + for content in contents: + id = content["id"] + if self.is_animation(id): + break + url_info = self.info_url(id) + info_content = req.get_json(url_info) + season = [{"season": s.get("name"), "air_date": s.get("air_date")} for s in info_content.get("seasons")] last_season = self.get_season(season) title_jp = info_content.get("original_name") From ec56f58e4765b43fe4c1747fff1fec592c30e5c5 Mon Sep 17 00:00:00 2001 From: WeijiangChen Date: Sun, 12 Mar 2023 21:56:04 +0800 Subject: [PATCH 2/9] =?UTF-8?q?=E5=BF=98=E8=AE=B0=E9=87=8D=E8=BD=BD?= =?UTF-8?q?=E4=BA=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- module/network/request_contents.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/module/network/request_contents.py b/module/network/request_contents.py index 631c32ca..cb689ea7 100644 --- a/module/network/request_contents.py +++ b/module/network/request_contents.py @@ -7,16 +7,14 @@ import re FILTER = "|".join(settings.rss_parser.filter) + @dataclass class TorrentInfo: name: str torrent_link: str -class RequestContent: - def __init__(self): - self._req = RequestURL() - +class RequestContent(RequestURL): # Mikanani RSS def get_torrents(self, _url: str) -> [TorrentInfo]: soup = self._req.get_content(_url) @@ -37,6 +35,3 @@ class RequestContent: # API JSON def get_json(self, _url) -> dict: return self._req.get_content(_url, content="json") - - def close_session(self): - self._req.close() From 106e4847c0634cda87b5d710eda47fc7881a02ae Mon Sep 17 00:00:00 2001 From: WeijiangChen Date: Sun, 12 Mar 2023 22:34:47 +0800 Subject: [PATCH 3/9] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E7=B1=BB=E9=87=8D?= =?UTF-8?q?=E8=BD=BD=E5=BF=98=E6=94=B9=E5=87=BD=E6=95=B0=E4=BA=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- module/network/request_contents.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/module/network/request_contents.py b/module/network/request_contents.py index cb689ea7..15b04c60 100644 --- a/module/network/request_contents.py +++ b/module/network/request_contents.py @@ -17,7 +17,7 @@ class TorrentInfo: class RequestContent(RequestURL): # Mikanani RSS def get_torrents(self, _url: str) -> [TorrentInfo]: - soup = self._req.get_content(_url) + soup = self.session.get_content(_url) torrent_titles = [item.title.string for item in soup.find_all("item")] torrent_urls = [item.get("url") for item in soup.find_all("enclosure")] torrents = [] @@ -27,11 +27,11 @@ class RequestContent(RequestURL): return torrents def get_torrent(self, _url) -> TorrentInfo: - soup = self._req.get_content(_url) + soup = self.session.get_content(_url) item = soup.find("item") enclosure = item.find("enclosure") return TorrentInfo(item.title.string, enclosure["url"]) # API JSON def get_json(self, _url) -> dict: - return self._req.get_content(_url, content="json") + return self.session.get_content(_url, content="json") From fa944a34b39ff77ecb79cfae079365ee2220d494 Mon Sep 17 00:00:00 2001 From: WeijiangChen Date: Sun, 12 Mar 2023 22:35:25 +0800 Subject: [PATCH 4/9] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E7=B1=BB=E9=87=8D?= =?UTF-8?q?=E8=BD=BD=E5=BF=98=E6=94=B9=E5=87=BD=E6=95=B0=E4=BA=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- module/network/request_contents.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/module/network/request_contents.py b/module/network/request_contents.py index 15b04c60..3c9506c8 100644 --- a/module/network/request_contents.py +++ b/module/network/request_contents.py @@ -17,7 +17,7 @@ class TorrentInfo: class RequestContent(RequestURL): # Mikanani RSS def get_torrents(self, _url: str) -> [TorrentInfo]: - soup = self.session.get_content(_url) + soup = self.get_content(_url) torrent_titles = [item.title.string for item in soup.find_all("item")] torrent_urls = [item.get("url") for item in soup.find_all("enclosure")] torrents = [] @@ -27,11 +27,11 @@ class RequestContent(RequestURL): return torrents def get_torrent(self, _url) -> TorrentInfo: - soup = self.session.get_content(_url) + soup = self.get_content(_url) item = soup.find("item") enclosure = item.find("enclosure") return TorrentInfo(item.title.string, enclosure["url"]) # API JSON def get_json(self, _url) -> dict: - return self.session.get_content(_url, content="json") + return self.get_content(_url, content="json") From aee3a3be3312fd10c3552fe6a0354a7e00b62d1c Mon Sep 17 00:00:00 2001 From: WeijiangChen Date: Sun, 12 Mar 2023 22:43:06 +0800 Subject: [PATCH 5/9] =?UTF-8?q?=E6=98=8E=E7=A1=AEget=20json=E4=B8=8Eget=20?= =?UTF-8?q?xml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- module/network/request_contents.py | 11 ++++++++--- module/network/request_url.py | 8 -------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/module/network/request_contents.py b/module/network/request_contents.py index 3c9506c8..6a0bfbf6 100644 --- a/module/network/request_contents.py +++ b/module/network/request_contents.py @@ -1,5 +1,7 @@ from dataclasses import dataclass +from bs4 import BeautifulSoup + from .request_url import RequestURL from module.conf import settings @@ -17,7 +19,7 @@ class TorrentInfo: class RequestContent(RequestURL): # Mikanani RSS def get_torrents(self, _url: str) -> [TorrentInfo]: - soup = self.get_content(_url) + soup = self.get_xml(_url) torrent_titles = [item.title.string for item in soup.find_all("item")] torrent_urls = [item.get("url") for item in soup.find_all("enclosure")] torrents = [] @@ -27,11 +29,14 @@ class RequestContent(RequestURL): return torrents def get_torrent(self, _url) -> TorrentInfo: - soup = self.get_content(_url) + soup = self.get_xml(_url) item = soup.find("item") enclosure = item.find("enclosure") return TorrentInfo(item.title.string, enclosure["url"]) + def get_xml(self, url): + return BeautifulSoup(self.get_url(url).text, "xml") + # API JSON def get_json(self, _url) -> dict: - return self.get_content(_url, content="json") + return self.get_url(_url).json() diff --git a/module/network/request_url.py b/module/network/request_url.py index 4f38734b..5dbd30e6 100644 --- a/module/network/request_url.py +++ b/module/network/request_url.py @@ -5,8 +5,6 @@ import socket import socks import logging -from bs4 import BeautifulSoup - from module.conf import settings logger = logging.getLogger(__name__) @@ -37,12 +35,6 @@ class RequestURL: logger.debug(e) break - def get_content(self, url, content="xml"): - if content == "xml": - return BeautifulSoup(self.get_url(url).text, content) - elif content == "json": - return self.get_url(url).json() - def __enter__(self): self.session = requests.Session() if settings.proxy.enable: From f4d4a06f371d431dbefb8fcb65ecaca5e69fc152 Mon Sep 17 00:00:00 2001 From: WeijiangChen Date: Sun, 12 Mar 2023 22:45:55 +0800 Subject: [PATCH 6/9] =?UTF-8?q?=E6=BC=8F=E6=94=B9=E4=BA=86=E4=B8=80?= =?UTF-8?q?=E4=B8=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- module/core/eps_complete.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/module/core/eps_complete.py b/module/core/eps_complete.py index 6b1ede72..87c2a3c9 100644 --- a/module/core/eps_complete.py +++ b/module/core/eps_complete.py @@ -57,7 +57,8 @@ class FullSeasonGet: self.download_eps(data, download_client) def download_collection(self, data, link, download_client: DownloadClient): - torrents = self._get_rss.get_torrents(link) + with RequestContent() as req: + torrents = req.get_torrents(link) downloads = self.collect_season_torrents(data, torrents) logger.info(f"Starting download {data.get('official_title')}") for download in downloads: From e9f493863b0efc2bf8c51c2b268afcebc17df703 Mon Sep 17 00:00:00 2001 From: WeijiangChen Date: Mon, 13 Mar 2023 00:17:30 +0800 Subject: [PATCH 7/9] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=8E=A8=E9=80=81?= =?UTF-8?q?=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- module/core/renamer.py | 3 +++ module/network/__init__.py | 6 +----- module/network/notification.py | 34 +++++++++++++++++++++++++++++++++- 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/module/core/renamer.py b/module/core/renamer.py index 98d5c2e3..bd31f99e 100644 --- a/module/core/renamer.py +++ b/module/core/renamer.py @@ -8,6 +8,7 @@ from .download_client import DownloadClient from module.conf import settings from module.parser import TitleParser +from ..network import PostNotification, FtqqNotification logger = logging.getLogger(__name__) @@ -53,6 +54,7 @@ class Renamer: return path_name, season, folder_name, suffix, download_path def run(self): + notification = FtqqNotification() recent_info, torrent_count = self.get_torrent_info() rename_count = 0 for info in recent_info: @@ -68,6 +70,7 @@ class Renamer: old_name = info.content_path.replace(info.save_path, "") self.client.rename_torrent_file(torrent_hash, new_name, old_name, new_name) rename_count += 1 + notification.send_msg(f"《{name[:10]}》缓存成功", f"[Auto Bangumi]《{name}》缓存成功") else: continue except Exception as e: diff --git a/module/network/__init__.py b/module/network/__init__.py index 269d54ac..326efdf1 100644 --- a/module/network/__init__.py +++ b/module/network/__init__.py @@ -1,6 +1,2 @@ from .request_contents import RequestContent -from .notification import PostNotification - - - - +from .notification import PostNotification, FtqqNotification diff --git a/module/network/notification.py b/module/network/notification.py index 06086f0b..75dc141c 100644 --- a/module/network/notification.py +++ b/module/network/notification.py @@ -1,8 +1,13 @@ +import logging + import requests from module.conf import settings +logger = logging.getLogger(__name__) + + class PostNotification: def __init__(self): self.token = settings.notification_token @@ -11,4 +16,31 @@ class PostNotification: def ifttt_post(self, message): url = self.notification_url(message) response = requests.get(url) - return response.status_code == 200 \ No newline at end of file + return response.status_code == 200 + + +class FtqqNotification: + """Server酱推送""" + + def __init__(self): + self.token = settings.notification.token + self.notification_url = f"https://sctapi.ftqq.com/{self.token}.send" + + def send_msg(self, title: str, desp: str) -> bool: + data = { + "title": title, + "desp": desp, + } + try: + resp = requests.post(self.notification_url, json=data, timeout=3) + resp.raise_for_status() + except requests.RequestException as e: + logging.error("[FtqqNotification] send fail, error: %s" % e) + return False + return True + + +if __name__ == '__main__': + name = "勇者、辞职不干了" + notification = FtqqNotification() + notification.send_msg(f"《{name[:10]}》缓存成功", f"[Auto Bangumi]《{name}》缓存成功") From dc8347ffaaa247b162e8044cb8acc047896f8bd0 Mon Sep 17 00:00:00 2001 From: WeijiangChen Date: Mon, 13 Mar 2023 00:28:40 +0800 Subject: [PATCH 8/9] =?UTF-8?q?=E9=87=8D=E5=91=BD=E5=90=8D=E6=8E=A8?= =?UTF-8?q?=E9=80=81=E5=87=BD=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- module/core/renamer.py | 4 ++-- module/network/__init__.py | 2 +- module/network/notification.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/module/core/renamer.py b/module/core/renamer.py index bd31f99e..7cd681f2 100644 --- a/module/core/renamer.py +++ b/module/core/renamer.py @@ -8,7 +8,7 @@ from .download_client import DownloadClient from module.conf import settings from module.parser import TitleParser -from ..network import PostNotification, FtqqNotification +from ..network import PostNotification, ServerChanNotification logger = logging.getLogger(__name__) @@ -54,7 +54,7 @@ class Renamer: return path_name, season, folder_name, suffix, download_path def run(self): - notification = FtqqNotification() + notification = ServerChanNotification() recent_info, torrent_count = self.get_torrent_info() rename_count = 0 for info in recent_info: diff --git a/module/network/__init__.py b/module/network/__init__.py index 326efdf1..d4589016 100644 --- a/module/network/__init__.py +++ b/module/network/__init__.py @@ -1,2 +1,2 @@ from .request_contents import RequestContent -from .notification import PostNotification, FtqqNotification +from .notification import PostNotification, ServerChanNotification diff --git a/module/network/notification.py b/module/network/notification.py index 75dc141c..1c0bb119 100644 --- a/module/network/notification.py +++ b/module/network/notification.py @@ -19,7 +19,7 @@ class PostNotification: return response.status_code == 200 -class FtqqNotification: +class ServerChanNotification: """Server酱推送""" def __init__(self): @@ -35,12 +35,12 @@ class FtqqNotification: resp = requests.post(self.notification_url, json=data, timeout=3) resp.raise_for_status() except requests.RequestException as e: - logging.error("[FtqqNotification] send fail, error: %s" % e) + logging.error("[ServerChanNotification] send fail, error: %s" % e) return False return True if __name__ == '__main__': name = "勇者、辞职不干了" - notification = FtqqNotification() + notification = ServerChanNotification() notification.send_msg(f"《{name[:10]}》缓存成功", f"[Auto Bangumi]《{name}》缓存成功") From b901319fb151efee9fb30692f7ada7818246177e Mon Sep 17 00:00:00 2001 From: WeijiangChen Date: Mon, 13 Mar 2023 20:50:59 +0800 Subject: [PATCH 9/9] =?UTF-8?q?=E5=A6=82=E6=9E=9C=E9=80=9A=E7=9F=A5?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=E6=B2=A1=E5=BC=80=E5=90=AF=EF=BC=8C=E4=B8=8D?= =?UTF-8?q?=E5=90=AF=E7=94=A8=E9=80=9A=E7=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- module/network/notification.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/module/network/notification.py b/module/network/notification.py index 1c0bb119..834f9a68 100644 --- a/module/network/notification.py +++ b/module/network/notification.py @@ -27,6 +27,8 @@ class ServerChanNotification: self.notification_url = f"https://sctapi.ftqq.com/{self.token}.send" def send_msg(self, title: str, desp: str) -> bool: + if not settings.notification.enable: + return False data = { "title": title, "desp": desp,