From 4a5eba53d94bebc5a643ca22aaa47f4ff01f59aa Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Mon, 13 Jun 2022 01:02:55 +0800 Subject: [PATCH] =?UTF-8?q?2.5.0-pre5=20=E6=9B=B4=E6=96=B0=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E7=9B=AE=E5=BD=95=EF=BC=8C=E4=BF=AE=E5=A4=8D=E5=A4=A7?= =?UTF-8?q?=E9=87=8Fbug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 2 +- .gitignore | 5 +- .idea/Bangumi_Auto_Rename.iml | 2 +- .idea/misc.xml | 2 +- Dockerfile | 4 +- auto_bangumi/app.py | 28 +-- auto_bangumi/bangumi_parser/fuzz_match.py | 2 +- auto_bangumi/bangumi_parser/parser.py | 4 +- auto_bangumi/conf/__init__.py | 0 auto_bangumi/{ => conf}/argument_parser.py | 0 auto_bangumi/{ => conf}/conf.py | 0 auto_bangumi/{ => conf}/const.py | 3 +- auto_bangumi/{ => conf}/log.py | 0 auto_bangumi/core/download_client.py | 7 +- auto_bangumi/core/renamer.py | 2 +- auto_bangumi/downloader/qb_downloader.py | 2 +- auto_bangumi/mikanani/__init__.py | 0 .../{core => mikanani}/eps_complete.py | 11 +- .../{core => mikanani}/rss_collector.py | 28 +-- auto_bangumi/network/__init__.py | 0 auto_bangumi/network/request.py | 46 +++++ auto_bangumi/web.py | 6 +- templates/index.html | 162 ++++++++++++++++++ templates/index.js | 5 + 24 files changed, 265 insertions(+), 56 deletions(-) create mode 100644 auto_bangumi/conf/__init__.py rename auto_bangumi/{ => conf}/argument_parser.py (100%) rename auto_bangumi/{ => conf}/conf.py (100%) rename auto_bangumi/{ => conf}/const.py (97%) rename auto_bangumi/{ => conf}/log.py (100%) create mode 100644 auto_bangumi/mikanani/__init__.py rename auto_bangumi/{core => mikanani}/eps_complete.py (85%) rename auto_bangumi/{core => mikanani}/rss_collector.py (77%) create mode 100644 auto_bangumi/network/__init__.py create mode 100644 auto_bangumi/network/request.py create mode 100644 templates/index.html create mode 100644 templates/index.js diff --git a/.dockerignore b/.dockerignore index a167b42c..8e05b4b9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -19,5 +19,5 @@ coverage.xml .pytest_cache .hypothesis -auto_bangumi/const_dev.py +auto_bangumi/conf/const_dev.py config/bangumi.json/config/bangumi.json diff --git a/.gitignore b/.gitignore index 451c94c8..5fa0a6bc 100644 --- a/.gitignore +++ b/.gitignore @@ -160,9 +160,10 @@ cython_debug/ #.idea/ # Custom -/auto_bangumi/const_dev.py +/auto_bangumi/conf/const_dev.py /config/bangumi.json /auto_bangumi/tester.py /resource/names.txt -/auto_bangumi/webui/ + + diff --git a/.idea/Bangumi_Auto_Rename.iml b/.idea/Bangumi_Auto_Rename.iml index f79e6193..1c2660df 100644 --- a/.idea/Bangumi_Auto_Rename.iml +++ b/.idea/Bangumi_Auto_Rename.iml @@ -4,7 +4,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index e863a931..37b833f1 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,4 +1,4 @@ - + \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index cfee8c25..43d92eaf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,6 +11,6 @@ ENV TZ=Asia/Shanghai ADD ./auto_bangumi /auto_bangumi ADD ./config /config +ADD ./templates /templates -CMD [ "python3", "app.py"] -CMD [ "uvicorn", "api:app", "--reload" ] +CMD [ "run.sh"] diff --git a/auto_bangumi/app.py b/auto_bangumi/app.py index c21113eb..3919afa4 100644 --- a/auto_bangumi/app.py +++ b/auto_bangumi/app.py @@ -1,16 +1,16 @@ -from ast import arg import os import time import logging from conf import settings -from argument_parser import parse -from log import setup_logger +from conf.argument_parser import parse +from conf.log import setup_logger from utils import json_config -from core.rss_collector import RSSCollector +from mikanani.rss_collector import RSSCollector from core.download_client import DownloadClient from core.renamer import Renamer +from network.request import RequestsURL logger = logging.getLogger(__name__) @@ -55,42 +55,48 @@ def show_info(): logger.info("Starting AutoBangumi...") - def run(): + # DEBUG 模式初始化 args = parse() if args.debug: try: - from const_dev import DEV_SETTINGS - import sys + from conf.const_dev import DEV_SETTINGS + settings.init(DEV_SETTINGS) except ModuleNotFoundError: logger.debug("Please copy `const_dev.py` to `const_dev.py` to use custom settings") - sys.path.append('/opt/homebrew/Caskroom/miniforge/base/envs/auto_bangumi/bin/python') - settings.init(DEV_SETTINGS) else: settings.init() + # 初始化 setup_logger() show_info() time.sleep(3) download_client = DownloadClient() + url_request = RequestsURL() download_client.init_downloader() if settings.rss_link is None: logger.error("Please add RIGHT RSS url.") quit() download_client.rss_feed() - rss_collector = RSSCollector() + rss_collector = RSSCollector(url_request) rename = Renamer(download_client) + # 主程序循环 while True: bangumi_data = load_data_file() try: + # 解析 RSS rss_collector.collect(bangumi_data) + # 历史剧集收集 if settings.enable_eps_complete: - download_client.eps_collect(bangumi_data["bangumi_info"]) + download_client.eps_collect(bangumi_data["bangumi_info"], url_request) + url_request.close() download_client.add_rules(bangumi_data["bangumi_info"], settings.rss_link) + # 首次等待 if bangumi_data["first_run"]: logger.info(f"Waiting for downloading torrents...") time.sleep(settings.first_sleep) bangumi_data["first_run"] = False save_data_file(bangumi_data) + # rename if settings.method != "none": rename.refresh() rename.run() diff --git a/auto_bangumi/bangumi_parser/fuzz_match.py b/auto_bangumi/bangumi_parser/fuzz_match.py index d0e8ffb3..924bc890 100644 --- a/auto_bangumi/bangumi_parser/fuzz_match.py +++ b/auto_bangumi/bangumi_parser/fuzz_match.py @@ -42,7 +42,7 @@ class FuzzMatch: if __name__ == "__main__": - from const_dev import DEV_SETTINGS + from conf.const_dev import DEV_SETTINGS settings.init(DEV_SETTINGS) f = FuzzMatch() name = "勇者、辞职不干了" diff --git a/auto_bangumi/bangumi_parser/parser.py b/auto_bangumi/bangumi_parser/parser.py index 1df585f0..7d7bc5cf 100644 --- a/auto_bangumi/bangumi_parser/parser.py +++ b/auto_bangumi/bangumi_parser/parser.py @@ -24,8 +24,8 @@ if __name__ == "__main__": import sys, os sys.path.append(os.path.dirname("..")) - from log import setup_logger - from const import BCOLORS + from conf.log import setup_logger + from conf.const import BCOLORS setup_logger() parser = Parser() diff --git a/auto_bangumi/conf/__init__.py b/auto_bangumi/conf/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/auto_bangumi/argument_parser.py b/auto_bangumi/conf/argument_parser.py similarity index 100% rename from auto_bangumi/argument_parser.py rename to auto_bangumi/conf/argument_parser.py diff --git a/auto_bangumi/conf.py b/auto_bangumi/conf/conf.py similarity index 100% rename from auto_bangumi/conf.py rename to auto_bangumi/conf/conf.py diff --git a/auto_bangumi/const.py b/auto_bangumi/conf/const.py similarity index 97% rename from auto_bangumi/const.py rename to auto_bangumi/conf/const.py index 6bd7d0c7..aefd996a 100644 --- a/auto_bangumi/const.py +++ b/auto_bangumi/conf/const.py @@ -45,7 +45,8 @@ ENV_TO_ATTR = { "AB_REMOVE_BAD_BT": ("remove_bad_torrent", lambda e: e.lower() in ("true", "1", "t")), "AB_FIRST_SLEEP": ("first_sleep", lambda e: float(e)), "AB_WEBUI_PORT": ("webui_port", lambda e: int(e)), - "AB_FUZZ_MATCH": ("enable_fuzz_match", lambda e: e.lower() in ("true", "1", "t")) + "AB_FUZZ_MATCH": ("enable_fuzz_match", lambda e: e.lower() in ("true", "1", "t")), + "AB_HTTP_PROXY": "http_proxy" } diff --git a/auto_bangumi/log.py b/auto_bangumi/conf/log.py similarity index 100% rename from auto_bangumi/log.py rename to auto_bangumi/conf/log.py diff --git a/auto_bangumi/core/download_client.py b/auto_bangumi/core/download_client.py index 6934e5d8..5ccfbe2d 100644 --- a/auto_bangumi/core/download_client.py +++ b/auto_bangumi/core/download_client.py @@ -8,7 +8,7 @@ from downloader.exceptions import ConflictError from conf import settings -from core.eps_complete import FullSeasonGet +from mikanani.eps_complete import FullSeasonGet logger = logging.getLogger(__name__) @@ -81,7 +81,7 @@ class DownloadClient: info["added"] = True logger.info("Finished.") - def eps_collect(self, bangumi_info): + def eps_collect(self, bangumi_info, request): logger.info("Start collecting past episodes.") for info in bangumi_info: if info["download_past"]: @@ -91,7 +91,8 @@ class DownloadClient: info["season"], info["subtitle"], info["source"], - info["dpi"] + info["dpi"], + request ).add_torrents_info() for download in downloads: self.client.torrents_add( diff --git a/auto_bangumi/core/renamer.py b/auto_bangumi/core/renamer.py index e0db38b5..c57fea9d 100644 --- a/auto_bangumi/core/renamer.py +++ b/auto_bangumi/core/renamer.py @@ -56,7 +56,7 @@ class Renamer: if __name__ == "__main__": - from const_dev import DEV_SETTINGS + from conf.const_dev import DEV_SETTINGS settings.init(DEV_SETTINGS) client = DownloadClient() rename = Renamer(client) diff --git a/auto_bangumi/downloader/qb_downloader.py b/auto_bangumi/downloader/qb_downloader.py index 3ddf7bc5..900510d4 100644 --- a/auto_bangumi/downloader/qb_downloader.py +++ b/auto_bangumi/downloader/qb_downloader.py @@ -75,7 +75,7 @@ class QbDownloader: if __name__ == "__main__": try: - from const_dev import DEV_SETTINGS + from conf.const_dev import DEV_SETTINGS except ModuleNotFoundError: logger.debug("Please copy `const_dev.py` to `const_dev.py` to use custom settings") settings.init(DEV_SETTINGS) diff --git a/auto_bangumi/mikanani/__init__.py b/auto_bangumi/mikanani/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/auto_bangumi/core/eps_complete.py b/auto_bangumi/mikanani/eps_complete.py similarity index 85% rename from auto_bangumi/core/eps_complete.py rename to auto_bangumi/mikanani/eps_complete.py index b1bcb10a..202105a1 100644 --- a/auto_bangumi/core/eps_complete.py +++ b/auto_bangumi/mikanani/eps_complete.py @@ -6,18 +6,20 @@ from bs4 import BeautifulSoup import logging from conf import settings +from network.request import RequestsURL logger = logging.getLogger(__name__) class FullSeasonGet: - def __init__(self, group, bangumi_name, season, sub, source, dpi): + def __init__(self, group, bangumi_name, season, sub, source, dpi , request: RequestsURL): self.bangumi_name = re.sub(settings.rule_name_re, " ", bangumi_name).strip() self.group = "" if group is None else group self.season = season self.subtitle = "" if sub is None else sub self.source = "" if source is None else source self.dpi = dpi + self._req = request def get_season_rss(self): if self.season == "S01": @@ -26,11 +28,8 @@ class FullSeasonGet: season = self.season search_str = re.sub(r"[\W_]", "+", f"{self.group} {self.bangumi_name} {season} {self.subtitle} {self.source} {self.dpi}") - season = requests.get( - f"https://mikanani.me/RSS/Search?searchstr={search_str}" - ) - soup = BeautifulSoup(season.content, "xml") - torrents = soup.find_all("enclosure") + season = self._req.get_url(search_str) + torrents = season.find_all("enclosure") return torrents def add_torrents_info(self): diff --git a/auto_bangumi/core/rss_collector.py b/auto_bangumi/mikanani/rss_collector.py similarity index 77% rename from auto_bangumi/core/rss_collector.py rename to auto_bangumi/mikanani/rss_collector.py index b1875737..b4e9dedd 100644 --- a/auto_bangumi/core/rss_collector.py +++ b/auto_bangumi/mikanani/rss_collector.py @@ -1,29 +1,19 @@ # -*- coding: UTF-8 -*- -import os import logging -import requests -from bs4 import BeautifulSoup from conf import settings from bangumi_parser.analyser.rss_parser import ParserLV2 from bangumi_parser.fuzz_match import FuzzMatch +from network.request import RequestsURL logger = logging.getLogger(__name__) class RSSCollector: - def __init__(self): + def __init__(self, request: RequestsURL): self._simple_analyser = ParserLV2() self._fuzz_match = FuzzMatch() - - def get_rss_info(self, rss_link): - try: - req = requests.get(rss_link, "utf-8") - rss = BeautifulSoup(req.text, "xml") - return rss - except Exception as e: - # logger.exception(e) - logger.error("ERROR with DNS/Connection.") + self._req = request def title_parser(self, title, fuzz_match=True): episode = self._simple_analyser.analyse(title) @@ -53,8 +43,8 @@ class RSSCollector: return episode, data, title_official def collect(self, bangumi_data): - rss = self.get_rss_info(settings.rss_link) - items = rss.find_all("item") + req = self._req.get_url(settings.rss_link) + items = req.find_all("item") for item in items: add = True name = item.title.string @@ -70,19 +60,19 @@ class RSSCollector: logger.info(f"Adding {title_official} Season {episode.season_info.number}") def collect_collection(self, rss_link): - rss = self.get_rss_info(rss_link) - item = rss.find("item") + req = self._req.get_url(rss_link) + item = req.find("item") title = item.title.string _, data, _ = self.title_parser(title, fuzz_match=False) return data if __name__ == "__main__": - from const_dev import DEV_SETTINGS + from conf.const_dev import DEV_SETTINGS from utils import json_config settings.init(DEV_SETTINGS) rss = RSSCollector() - info = json_config.load("/Users/Estrella/Developer/Bangumi_Auto_Collector/config/bangumi.json") + info = json_config.load("/config/bangumi.json") rss.collect(info) print(info) diff --git a/auto_bangumi/network/__init__.py b/auto_bangumi/network/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/auto_bangumi/network/request.py b/auto_bangumi/network/request.py new file mode 100644 index 00000000..4d0edcef --- /dev/null +++ b/auto_bangumi/network/request.py @@ -0,0 +1,46 @@ +import time + +import requests +import logging + +from bs4 import BeautifulSoup + +from conf import settings + +logger = logging.getLogger(__name__) + + +class RequestsURL: + def __init__(self): + self.session = requests.session() + if settings.http_proxy is not None: + self.proxy = { + "https": settings.http_proxy, + "http": settings.http_proxy + } + else: + self.proxy = None + + def get_url(self, url): + times = 1 + while times < 5: + try: + req = self.session.get(url, proxies=self.proxy) + return BeautifulSoup(req.text, "xml") + except Exception: + # logger.exception(e) + logger.error("ERROR with DNS/Connection.") + time.sleep(settings.connect_retry_interval) + times += 1 + + def close(self): + self.session.close() + + +if __name__ == "__main__": + network_req = RequestsURL() + req = network_req.get_url("https://mikanani.me/RSS/Classic") + print(req.find_all("item")) + network_req.close() + req = network_req.get_url("https://mikanani.me/RSS/Classic") + print(req.find_all("item")) \ No newline at end of file diff --git a/auto_bangumi/web.py b/auto_bangumi/web.py index 83964aaa..008f2a7f 100644 --- a/auto_bangumi/web.py +++ b/auto_bangumi/web.py @@ -1,5 +1,3 @@ -from typing import Union - import uvicorn from fastapi import FastAPI, Request from fastapi.responses import HTMLResponse @@ -7,7 +5,7 @@ from fastapi.templating import Jinja2Templates from pydantic import BaseModel import logging -from core.rss_collector import RSSCollector +from mikanani.rss_collector import RSSCollector from core.download_client import DownloadClient from conf import settings from utils import json_config @@ -68,7 +66,7 @@ class RSS(BaseModel): @app.post("/api/v1/subscriptions") async def receive(link: RSS): data = RSSCollector().collect_collection(link.link) - from const_dev import DEV_SETTINGS + from conf.const_dev import DEV_SETTINGS settings.init(DEV_SETTINGS) client = DownloadClient() client.add_collection_feed(link.link, item_path=data["title"]) diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 00000000..7bb7f8e4 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,162 @@ + + + + + + AutoBangumi + + + +
+
+
+
+ +
+
+
+
+
+

Configuration

+
+
+
+
+

基础配置

+ +
  • +
    +
    +
    +
    +

    RSS link

    +
    +
    + +
    +
    +
    +
    +
    • +
  • +
    +
    +
    +
    +

    HOST

    +
    +
    + +
    +
    +
    +
    +
    • +
  • +
    +
    +
    +
    +

    Username

    +
    +
    + +
    +
    +
    +
    +
    • +
  • +
    +
    +
    +
    +

    Password

    +
    +
    + +
    +
    +
    +
    +
    • +
  • +
    +
    +
    +
    +

    Download Path

    +
    +
    + +
    +
    +
    +
    +
    • + +     +
    +
    +
    +
    +

    进阶配置

    + +
  • +
    +
    +
    +

    字幕组 TAG

    +
    + + + + + + +
    +
    +
    +
    +
    • +
  • +
    +
    +
    +

    历史补全

    +
    + + + + + + +
    +
    +
    +
    +
    • +
  • +
    +
    +
    +

    坏种移除

    +
    + + + + + + +
    +
    +
    +
    +
    • +     +
    +
    +
    + + + \ No newline at end of file diff --git a/templates/index.js b/templates/index.js new file mode 100644 index 00000000..58e33404 --- /dev/null +++ b/templates/index.js @@ -0,0 +1,5 @@ +function postConfig(){ + var httpRequest = new XMLHttpRequest(); + httpRequest.open("POST", "autobangumi", ture) + +} \ No newline at end of file