From dd5c9187038a6737ed0cc983494a1413db05ac43 Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Sat, 12 Aug 2023 15:40:43 +0800 Subject: [PATCH] change: change rss item database, add more option for split rss link. --- backend/src/module/api/config.py | 3 +- backend/src/module/api/rss.py | 42 +------------ backend/src/module/checker/checker.py | 22 ++++++- backend/src/module/conf/config.py | 8 --- backend/src/module/core/program.py | 15 ++++- backend/src/module/core/rss_feed.py | 27 -------- backend/src/module/core/status.py | 4 ++ backend/src/module/models/config.py | 7 --- backend/src/module/models/rss.py | 6 +- backend/src/module/parser/title_parser.py | 6 +- backend/src/module/rss/analyser.py | 69 ++++++++++----------- backend/src/module/update/__init__.py | 4 +- backend/src/module/update/data_migration.py | 7 ++- backend/src/module/update/rss.py | 6 ++ backend/src/module/update/startup.py | 21 ++++--- 15 files changed, 104 insertions(+), 143 deletions(-) delete mode 100644 backend/src/module/core/rss_feed.py create mode 100644 backend/src/module/update/rss.py diff --git a/backend/src/module/api/config.py b/backend/src/module/api/config.py index e39e528c..777af52e 100644 --- a/backend/src/module/api/config.py +++ b/backend/src/module/api/config.py @@ -14,7 +14,7 @@ logger = logging.getLogger(__name__) async def get_config(current_user=Depends(get_current_user)): if not current_user: raise UNAUTHORIZED - return settings.dict() + return settings @router.patch("/update") @@ -24,6 +24,7 @@ async def update_config(config: Config, current_user=Depends(get_current_user)): try: settings.save(config_dict=config.dict()) settings.load() + update_rss() logger.info("Config updated") return {"message": "Success"} except Exception as e: diff --git a/backend/src/module/api/rss.py b/backend/src/module/api/rss.py index 85a9a442..e19ce2c9 100644 --- a/backend/src/module/api/rss.py +++ b/backend/src/module/api/rss.py @@ -1,6 +1,6 @@ from typing import Optional -from fastapi import APIRouter, Depends, HTTPException, status +from fastapi import APIRouter, Depends, status from fastapi.responses import JSONResponse from .response import u_response @@ -39,16 +39,6 @@ async def delete_rss(rss_id: int, current_user=Depends(get_current_user)): raise UNAUTHORIZED with RSSEngine() as engine: result = engine.rss.delete(rss_id) - if result: - return JSONResponse( - status_code=status.HTTP_200_OK, - content={"status": f"Success deleted {rss_id}"}, - ) - else: - return JSONResponse( - status_code=status.HTTP_400_BAD_REQUEST, - content={"status": f"Failed to delete {rss_id}"}, - ) @router.patch("/update/{rss_id}") @@ -59,16 +49,6 @@ async def update_rss( raise UNAUTHORIZED with RSSEngine() as engine: result = engine.rss.update(rss_id, data) - if result: - return JSONResponse( - status_code=status.HTTP_200_OK, - content={"status": f"Success updated {data.item_path}"}, - ) - else: - return JSONResponse( - status_code=status.HTTP_400_BAD_REQUEST, - content={"status": f"Failed to update {data.item_path}"}, - ) @router.get("/refresh/all") @@ -77,16 +57,6 @@ async def refresh_all(current_user=Depends(get_current_user)): raise UNAUTHORIZED with RSSEngine() as engine, DownloadClient() as client: response = engine.refresh_rss(client) - if response: - return JSONResponse( - status_code=status.HTTP_200_OK, - content={"status": f"Success refresh all rss"}, - ) - else: - return JSONResponse( - status_code=status.HTTP_400_BAD_REQUEST, - content={"status": f"Failed to refresh all rss"}, - ) @router.get("/refresh/{rss_id}") @@ -95,13 +65,3 @@ async def refresh_rss(rss_id: int, current_user=Depends(get_current_user)): raise UNAUTHORIZED with RSSEngine() as engine, DownloadClient() as client: response = engine.refresh_rss(client, rss_id) - if response: - return JSONResponse( - status_code=status.HTTP_200_OK, - content={"status": f"Success refresh {rss_id}"}, - ) - else: - return JSONResponse( - status_code=status.HTTP_400_BAD_REQUEST, - content={"status": f"Failed to refresh {rss_id}"}, - ) diff --git a/backend/src/module/checker/checker.py b/backend/src/module/checker/checker.py index 8966c46d..d1827a2c 100644 --- a/backend/src/module/checker/checker.py +++ b/backend/src/module/checker/checker.py @@ -1,4 +1,6 @@ -from module.conf import settings +import os + +from module.conf import settings, VERSION from module.downloader import DownloadClient from module.models import Config from module.network import RequestContent @@ -49,3 +51,21 @@ class Checker: return True else: return False + + @staticmethod + def check_version() -> bool: + if not os.path.exists("config/version.info"): + with open("config/version.info", "w+") as f: + f.writelines(VERSION) + return True + with open("config/version.info", "r+") as f: + legacy_version = f.readlines()[-1] + if VERSION == legacy_version: + return False + else: + f.writelines(VERSION) + return True + + +if __name__ == "__main__": + print(Checker().check_version()) diff --git a/backend/src/module/conf/config.py b/backend/src/module/conf/config.py index 00debcce..30d046af 100644 --- a/backend/src/module/conf/config.py +++ b/backend/src/module/conf/config.py @@ -53,14 +53,6 @@ class Settings(Config): self.__load_from_env() self.save() - @property - def rss_link(self) -> str: - if "://" not in self.rss_parser.custom_url: - return f"https://{self.rss_parser.custom_url}/RSS/MyBangumi?token={self.rss_parser.token}" - return ( - f"{self.rss_parser.custom_url}/RSS/MyBangumi?token={self.rss_parser.token}" - ) - def __load_from_env(self): config_dict = self.dict() for key, section in ENV_TO_ATTR.items(): diff --git a/backend/src/module/core/program.py b/backend/src/module/core/program.py index e926d0f1..8bcb1fa4 100644 --- a/backend/src/module/core/program.py +++ b/backend/src/module/core/program.py @@ -1,7 +1,7 @@ import logging from module.conf import VERSION, settings -from module.update import data_migration, start_up +from module.update import data_migration, database_migration, start_up, first_run from .sub_thread import RenameThread, RSSThread @@ -32,8 +32,8 @@ class Program(RenameThread, RSSThread): def startup(self): self.__start_info() - start_up(self.first_run) if self.first_run: + first_run() logger.info("First run detected, please configure the program in webui.") return {"status": "First run detected."} if self.legacy_data: @@ -41,6 +41,10 @@ class Program(RenameThread, RSSThread): "Legacy data detected, starting data migration, please wait patiently." ) data_migration() + elif self.version_update: + # Update database + database_migration() + logger.info("Database updated.") self.start() def start(self): @@ -71,3 +75,10 @@ class Program(RenameThread, RSSThread): self.stop() self.start() return {"status": "Program restarted."} + + def update_database(self): + if not self.version_update: + return {"status": "No update found."} + else: + start_up(True) + return {"status": "Database updated."} diff --git a/backend/src/module/core/rss_feed.py b/backend/src/module/core/rss_feed.py deleted file mode 100644 index 2dae9d8c..00000000 --- a/backend/src/module/core/rss_feed.py +++ /dev/null @@ -1,27 +0,0 @@ -import logging - -from module.conf import settings -from module.downloader import DownloadClient - -logger = logging.getLogger(__name__) - - -def add_rss_feed(): - with DownloadClient() as client: - # Check Feed if exists - add = True - remove = False - feeds = client.get_rss_feed() - for item_path, value in feeds.items(): - if value.url == settings.rss_link: - add = False - break - elif item_path == "Mikan_RSS": - remove = True - if remove: - client.remove_rss_feed("Mikan_RSS") - logger.info("Remove Old RSS Feed: Mikan_RSS") - # Add Feed - if add: - client.add_rss_feed(settings.rss_link) - logger.info(f"Add RSS Feed: {settings.rss_link}") diff --git a/backend/src/module/core/status.py b/backend/src/module/core/status.py index 9d98daaf..5eadb60d 100644 --- a/backend/src/module/core/status.py +++ b/backend/src/module/core/status.py @@ -52,3 +52,7 @@ class ProgramStatus(Checker): @property def legacy_data(self): return LEGACY_DATA_PATH.exists() + + @property + def version_update(self): + return self.check_version() diff --git a/backend/src/module/models/config.py b/backend/src/module/models/config.py index a1169ad2..c282908a 100644 --- a/backend/src/module/models/config.py +++ b/backend/src/module/models/config.py @@ -30,16 +30,9 @@ class Downloader(BaseModel): class RSSParser(BaseModel): enable: bool = Field(True, description="Enable RSS parser") type: str = Field("mikan", description="RSS parser type") - token_: str = Field("token", alias="token", description="RSS parser token") - custom_url: str = Field("mikanani.me", description="Custom RSS host url") - parser_type: str = Field("parser", description="Parser type") filter: list[str] = Field(["720", r"\d+-\d"], description="Filter") language: str = "zh" - @property - def token(self): - return expandvars(self.token_) - class BangumiManage(BaseModel): enable: bool = Field(True, description="Enable bangumi manage") diff --git a/backend/src/module/models/rss.py b/backend/src/module/models/rss.py index aa98f028..d05253da 100644 --- a/backend/src/module/models/rss.py +++ b/backend/src/module/models/rss.py @@ -7,6 +7,7 @@ class RSSItem(SQLModel, table=True): item_path: str = Field("example path", alias="item_path") url: str = Field("https://mikanani.me", alias="url") combine: bool = Field(True, alias="combine") + parser: str = Field("mikan", alias="parser") enabled: bool = Field(True, alias="enabled") @@ -14,8 +15,5 @@ class RSSUpdate(SQLModel): item_path: Optional[str] = Field("example path", alias="item_path") url: Optional[str] = Field("https://mikanani.me", alias="url") combine: Optional[bool] = Field(True, alias="combine") + parser: Optional[str] = Field("mikan", alias="parser") enabled: Optional[bool] = Field(True, alias="enabled") - - - - diff --git a/backend/src/module/parser/title_parser.py b/backend/src/module/parser/title_parser.py index c2b33fe2..6214bb55 100644 --- a/backend/src/module/parser/title_parser.py +++ b/backend/src/module/parser/title_parser.py @@ -3,7 +3,7 @@ import logging from module.conf import settings from module.models import Bangumi -from .analyser import raw_parser, tmdb_parser, torrent_parser +from .analyser import raw_parser, tmdb_parser, torrent_parser, mikan_parser logger = logging.getLogger(__name__) @@ -79,3 +79,7 @@ class TitleParser: logger.debug(e) logger.warning(f"Cannot parse {raw}.") return None + + @staticmethod + def mikan_parser(homepage: str) -> tuple[str, str]: + return mikan_parser(homepage) diff --git a/backend/src/module/rss/analyser.py b/backend/src/module/rss/analyser.py index 768ead70..d5cbc062 100644 --- a/backend/src/module/rss/analyser.py +++ b/backend/src/module/rss/analyser.py @@ -4,7 +4,7 @@ import re from .engine import RSSEngine from module.conf import settings -from module.models import Bangumi, Torrent +from module.models import Bangumi, Torrent, RSSItem from module.network import RequestContent from module.parser import TitleParser @@ -12,19 +12,21 @@ logger = logging.getLogger(__name__) class RSSAnalyser(TitleParser): - def official_title_parser(self, data: Bangumi, mikan_title: str): - if settings.rss_parser.parser_type == "mikan": - data.official_title = mikan_title if mikan_title else data.official_title - elif settings.rss_parser.parser_type == "tmdb": - tmdb_title, season, year = self.tmdb_parser( - data.official_title, data.season, settings.rss_parser.language + def official_title_parser(self, bangumi: Bangumi, rss: RSSItem, torrent: Torrent): + if rss.parser == "mikan": + bangumi.official_title, bangumi.poster_link = self.mikan_parser( + torrent.homepage ) - data.official_title = tmdb_title - data.year = year - data.season = season + elif rss.parser == "tmdb": + tmdb_title, season, year = self.tmdb_parser( + bangumi.official_title, bangumi.season, settings.rss_parser.language + ) + bangumi.official_title = tmdb_title + bangumi.year = year + bangumi.season = season else: pass - data.official_title = re.sub(r"[/:.\\]", " ", data.official_title) + bangumi.official_title = re.sub(r"[/:.\\]", " ", bangumi.official_title) @staticmethod def get_rss_torrents(rss_link: str, full_parse: bool = True) -> list[Torrent]: @@ -36,42 +38,35 @@ class RSSAnalyser(TitleParser): return rss_torrents def torrents_to_data( - self, torrents: list, rss_link: str, full_parse: bool = True + self, torrents: list[Torrent], rss: RSSItem, full_parse: bool = True ) -> list: new_data = [] for torrent in torrents: - data = self.raw_parser(raw=torrent.name) - if data and data.title_raw not in [i.title_raw for i in new_data]: - with RequestContent() as req: - poster_link, mikan_title = req.get_mikan_info(torrent.homepage) - data.poster_link = poster_link - data.rss_link = rss_link - self.official_title_parser(data, mikan_title) + bangumi = self.raw_parser(raw=torrent.name) + if bangumi and bangumi.title_raw not in [i.title_raw for i in new_data]: + self.official_title_parser(bangumi=bangumi, rss=rss, torrent=torrent) if not full_parse: - return [data] - new_data.append(data) - logger.debug(f"[RSS] New title found: {data.official_title}") + return [bangumi] + new_data.append(bangumi) + logger.debug(f"[RSS] New title found: {bangumi.official_title}") return new_data - def torrent_to_data(self, torrent: Torrent) -> Bangumi: - data = self.raw_parser(raw=torrent.name) - if data: - with RequestContent() as req: - poster_link, mikan_title = req.get_mikan_info(torrent.homepage) - data.poster_link = poster_link - self.official_title_parser(data, mikan_title) - return data + def torrent_to_data(self, torrent: Torrent, rss: RSSItem) -> Bangumi: + bangumi = self.raw_parser(raw=torrent.name) + if bangumi: + self.official_title_parser(bangumi=bangumi, rss=rss, torrent=torrent) + return bangumi def rss_to_data( - self, rss_link: str, engine: RSSEngine, full_parse: bool = True + self, rss: RSSItem, engine: RSSEngine, full_parse: bool = True ) -> list[Bangumi]: - rss_torrents = self.get_rss_torrents(rss_link, full_parse) - torrents_to_add = engine.bangumi.match_list(rss_torrents, rss_link) + rss_torrents = self.get_rss_torrents(rss.url, full_parse) + torrents_to_add = engine.bangumi.match_list(rss_torrents, rss.url) if not torrents_to_add: logger.debug("[RSS] No new title has been found.") return [] # New List - new_data = self.torrents_to_data(torrents_to_add, rss_link, full_parse) + new_data = self.torrents_to_data(torrents_to_add, rss, full_parse) if new_data: # Add to database engine.bangumi.add_all(new_data) @@ -79,9 +74,9 @@ class RSSAnalyser(TitleParser): else: return [] - def link_to_data(self, link: str) -> Bangumi: - torrents = self.get_rss_torrents(link, False) + def link_to_data(self, rss: RSSItem) -> Bangumi: + torrents = self.get_rss_torrents(rss.url, False) for torrent in torrents: - data = self.torrent_to_data(torrent) + data = self.torrent_to_data(torrent, rss) if data: return data diff --git a/backend/src/module/update/__init__.py b/backend/src/module/update/__init__.py index 2f4148c6..1c991151 100644 --- a/backend/src/module/update/__init__.py +++ b/backend/src/module/update/__init__.py @@ -1,2 +1,2 @@ -from .data_migration import data_migration -from .startup import start_up +from .data_migration import data_migration, database_migration +from .startup import start_up, first_run diff --git a/backend/src/module/update/data_migration.py b/backend/src/module/update/data_migration.py index ed5a69b7..e007e94a 100644 --- a/backend/src/module/update/data_migration.py +++ b/backend/src/module/update/data_migration.py @@ -14,8 +14,11 @@ def data_migration(): for info in infos: new_data.append(Bangumi(**info, rss_link=[rss_link])) with RSSEngine() as engine: - engine.create_table() engine.bangumi.add_all(new_data) - engine.user.add_default_user() engine.add_rss(rss_link) LEGACY_DATA_PATH.unlink(missing_ok=True) + + +def database_migration(): + with RSSEngine() as engine: + engine.migrate() diff --git a/backend/src/module/update/rss.py b/backend/src/module/update/rss.py new file mode 100644 index 00000000..59e3e59c --- /dev/null +++ b/backend/src/module/update/rss.py @@ -0,0 +1,6 @@ +from module.rss import RSSEngine + + +def update_main_rss(rss_link: str): + with RSSEngine() as engine: + engine.add_rss(rss_link, "main", True) diff --git a/backend/src/module/update/startup.py b/backend/src/module/update/startup.py index b2a97e2f..c31b8ee4 100644 --- a/backend/src/module/update/startup.py +++ b/backend/src/module/update/startup.py @@ -6,18 +6,19 @@ from module.conf import settings logger = logging.getLogger(__name__) -def start_up(first_run): +def start_up(): with RSSEngine() as engine: engine.create_table() engine.user.add_default_user() - if not first_run: - main_rss = engine.rss.search_id(1) - if not main_rss: - engine.add_rss(settings.rss_link, name="Mikan RSS", combine=True) - elif main_rss.url != settings.rss_link: - main_rss.url = settings.rss_link - engine.rss.update(1, main_rss) + main_rss = engine.rss.search_id(1) + if not main_rss: + engine.add_rss(settings.rss_link, name="Mikan RSS", combine=True) + elif main_rss.url != settings.rss_link: + main_rss.url = settings.rss_link + engine.rss.update(1, main_rss) -if __name__ == "__main__": - start_up(False) +def first_run(): + with RSSEngine() as engine: + engine.create_table() + engine.user.add_default_user()