From adbe3a7aae503b3e2abd6f59d134ed8cd607604d Mon Sep 17 00:00:00 2001 From: estrella Date: Fri, 16 Jun 2023 21:10:35 +0800 Subject: [PATCH 1/9] feat: add rss engine --- backend/src/module/database/__init__.py | 2 + backend/src/module/database/torrent.py | 49 ++++++------ backend/src/module/models/__init__.py | 4 +- backend/src/module/models/rss.py | 19 +++-- .../src/module/network/request_contents.py | 9 +++ backend/src/module/network/site/mikan.py | 4 + backend/src/module/rss/engine.py | 75 +++++++++++++++++++ backend/src/module/rss/poller.py | 26 ------- 8 files changed, 129 insertions(+), 59 deletions(-) create mode 100644 backend/src/module/rss/engine.py delete mode 100644 backend/src/module/rss/poller.py diff --git a/backend/src/module/database/__init__.py b/backend/src/module/database/__init__.py index c5d59b08..cd90685d 100644 --- a/backend/src/module/database/__init__.py +++ b/backend/src/module/database/__init__.py @@ -1 +1,3 @@ from .bangumi import BangumiDatabase +from .rss import RSSDatabase +from .torrent import TorrentDatabase \ No newline at end of file diff --git a/backend/src/module/database/torrent.py b/backend/src/module/database/torrent.py index bbdf0f8c..2821182f 100644 --- a/backend/src/module/database/torrent.py +++ b/backend/src/module/database/torrent.py @@ -1,17 +1,24 @@ import logging -from .connector import DataConnector +from .orm import Connector +from module.models import TorrentData +from module.conf import DATA_PATH logger = logging.getLogger(__name__) -class TorrentDatabase(DataConnector): - def update_table(self): - table_name = "torrent" - db_data = self.__data_to_db() - self._update_table(table_name, db_data) +class TorrentDatabase(Connector): + def __init__(self, database: str = DATA_PATH): + super().__init__( + table_name="torrent", + data=TorrentData().dict(), + database=DATA_PATH + ) - def __data_to_db(self, data: SaveTorrent): + def update_table(self): + self.update.table() + + def __data_to_db(self, data: TorrentData) -> dict: db_data = data.dict() for key, value in db_data.items(): if isinstance(value, bool): @@ -20,28 +27,18 @@ class TorrentDatabase(DataConnector): db_data[key] = ",".join(value) return db_data - def __db_to_data(self, db_data: dict): + def __db_to_data(self, db_data: dict) -> TorrentData: for key, item in db_data.items(): if isinstance(item, int): - if key not in ["id", "offset", "season", "year"]: - db_data[key] = bool(item) + db_data[key] = bool(item) elif key in ["filter", "rss_link"]: db_data[key] = item.split(",") - return SaveTorrent(**db_data) + return TorrentData(**db_data) - def if_downloaded(self, torrent_url: str, torrent_name: str) -> bool: - self._cursor.execute( - "SELECT * FROM torrent WHERE torrent_url = ? OR torrent_name = ?", - (torrent_url, torrent_name), - ) - return bool(self._cursor.fetchone()) + def insert_many(self, data_list: list[TorrentData]): + dict_datas = [self.__data_to_db(data) for data in data_list] + self.insert.many(dict_datas) - def insert(self, data: SaveTorrent): - db_data = self.__data_to_db(data) - columns = ", ".join(db_data.keys()) - values = ", ".join([f":{key}" for key in db_data.keys()]) - self._cursor.execute( - f"INSERT INTO torrent ({columns}) VALUES ({values})", db_data - ) - logger.debug(f"Add {data.torrent_name} into database.") - self._conn.commit() + def get_all(self) -> list[TorrentData]: + dict_datas = self.select.all() + return [self.__db_to_data(data) for data in dict_datas] diff --git a/backend/src/module/models/__init__.py b/backend/src/module/models/__init__.py index a73f18ed..c2a2599e 100644 --- a/backend/src/module/models/__init__.py +++ b/backend/src/module/models/__init__.py @@ -1,5 +1,5 @@ -from .bangumi import * +from .bangumi import BangumiData from .config import Config -from .rss import RSSTorrents +from .rss import RSSItem, TorrentData from .torrent import EpisodeFile, SubtitleFile, TorrentBase from .user import UserLogin diff --git a/backend/src/module/models/rss.py b/backend/src/module/models/rss.py index c763c4c5..66f38292 100644 --- a/backend/src/module/models/rss.py +++ b/backend/src/module/models/rss.py @@ -1,9 +1,18 @@ from pydantic import BaseModel, Field -class RSSTorrents(BaseModel): - name: str = Field(..., alias="item_path") +class RSSItem(BaseModel): + id: int = Field(0, alias="id", title="id") + item_path: str = Field("example path", alias="item_path") + url: str = Field("https://mikanani.me", alias="url") + combine: bool = Field(True, alias="combine") + enabled: bool = Field(True, alias="enabled") + + +class TorrentData(BaseModel): + id: int = Field(0, alias="id") + name: str = Field(..., alias="name") url: str = Field(..., alias="url") - analyze: bool = Field(..., alias="analyze") - enabled: bool = Field(..., alias="enabled") - torrents: list[str] = Field(..., alias="torrents") + matched: bool = Field(..., alias="matched") + downloaded: bool = Field(..., alias="downloaded") + save_path: str = Field(..., alias="save_path") diff --git a/backend/src/module/network/request_contents.py b/backend/src/module/network/request_contents.py index fcef7774..7f00a1d1 100644 --- a/backend/src/module/network/request_contents.py +++ b/backend/src/module/network/request_contents.py @@ -95,3 +95,12 @@ class RequestContent(RequestURL): def check_connection(self, _url): return self.check_url(_url) + + def get_rss_title(self, _url): + soup = self.get_xml(_url) + return soup.find("title").text + + +if __name__ == '__main__': + with RequestContent() as req: + req.get_xml("https://mikanani.me/RSS/Classic") diff --git a/backend/src/module/network/site/mikan.py b/backend/src/module/network/site/mikan.py index 0ad4314b..ce6595e7 100644 --- a/backend/src/module/network/site/mikan.py +++ b/backend/src/module/network/site/mikan.py @@ -7,3 +7,7 @@ def mikan_parser(soup): torrent_urls.append(item.find("enclosure").attrib["url"]) torrent_homepage.append(item.find("link").text) return torrent_titles, torrent_urls, torrent_homepage + + +def mikan_title(soup): + return soup.find("title").text diff --git a/backend/src/module/rss/engine.py b/backend/src/module/rss/engine.py new file mode 100644 index 00000000..8befad0c --- /dev/null +++ b/backend/src/module/rss/engine.py @@ -0,0 +1,75 @@ +import re + +from module.database import RSSDatabase, BangumiDatabase, TorrentDatabase +from module.models import BangumiData, RSSItem, TorrentData +from module.network import RequestContent, TorrentInfo + + +class RSSEngine(RequestContent): + + @staticmethod + def _get_rss_items() -> list[RSSItem]: + with RSSDatabase() as db: + return db.get_all() + + @staticmethod + def _get_bangumi_data(rss_link: str) -> list[BangumiData]: + with BangumiDatabase() as db: + return db.get_rss_data(rss_link) + + def add_rss(self, rss_link: str, name: str, combine: bool): + if not name: + name = self.get_rss_title(rss_link) + insert_data = RSSItem(item_path=name, url=rss_link, combine=combine) + with RSSDatabase() as db: + db.insert_one(insert_data) + + def pull_rss(self, rss_item: RSSItem) -> list[TorrentInfo]: + torrents = self.get_torrents(rss_item.url) + return torrents + + @staticmethod + def match_torrent(torrent: TorrentInfo) -> TorrentData | None: + with BangumiDatabase() as db: + bangumi_data = db.match_torrent(torrent.name) + if bangumi_data: + _filter = "|".join(bangumi_data.filter) + if re.search(_filter, torrent.name): + return None + else: + return TorrentData( + name=torrent.name, + url=torrent.torrent_link, + ) + return None + + @staticmethod + def filter_torrent(torrents: list[TorrentInfo]) -> list[TorrentInfo]: + with TorrentDatabase() as db: + in_db_torrents = db.get_all() + in_db_torrents = [x.name for x in in_db_torrents] + i = 0 + while i < len(torrents): + torrent = torrents[i] + if torrent.name in in_db_torrents: + torrents.pop(i) + i += 1 + return torrents + + def run(self): + # Get All RSS Items + rss_items: list[RSSItem] = self._get_rss_items() + # From RSS Items, get all torrents + for rss_item in rss_items: + torrents = self.get_torrents(rss_item.url) + self.filter_torrent(torrents) + # Get all enabled bangumi data + matched_torrents = [] + for torrent in torrents: + matched_torrent = self.match_torrent(torrent) + if matched_torrent: + matched_torrents.append(matched_torrent) + # Add to database + with TorrentDatabase() as db: + db.insert_many(matched_torrents) + return matched_torrents diff --git a/backend/src/module/rss/poller.py b/backend/src/module/rss/poller.py deleted file mode 100644 index 90a81cb0..00000000 --- a/backend/src/module/rss/poller.py +++ /dev/null @@ -1,26 +0,0 @@ -import re - -from module.database import RSSDatabase -from module.models import BangumiData, RSSTorrents -from module.network import RequestContent, TorrentInfo - - -class RSSPoller(RSSDatabase): - @staticmethod - def polling(rss_link, req: RequestContent) -> list[TorrentInfo]: - return req.get_torrents(rss_link) - - @staticmethod - def filter_torrent(data: BangumiData, torrent: TorrentInfo) -> bool: - if data.title_raw in torrent.name: - _filter = "|".join(data.filter) - if not re.search(_filter, torrent.name): - return True - else: - return False - - def foo(self): - rss_datas: list[RSSTorrents] = self.get_rss_data() - with RequestContent() as req: - for rss_data in rss_datas: - self.polling(rss_data.url, req) From b4204cadca241cffde864ef6ea7b6a00907590a4 Mon Sep 17 00:00:00 2001 From: estrella Date: Wed, 21 Jun 2023 00:18:32 +0800 Subject: [PATCH 2/9] feat: add rss engine --- backend/src/module/database/rss.py | 44 ++++++++++++++++++++++++++++++ backend/src/module/rss/engine.py | 6 +++- 2 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 backend/src/module/database/rss.py diff --git a/backend/src/module/database/rss.py b/backend/src/module/database/rss.py new file mode 100644 index 00000000..8c86ef66 --- /dev/null +++ b/backend/src/module/database/rss.py @@ -0,0 +1,44 @@ +from .orm import Connector + +from module.models import RSSItem +from module.conf import DATA_PATH + + +class RSSDatabase(Connector): + def __init__(self, database: str = DATA_PATH): + super().__init__( + table_name="RSSItem", + data=RSSItem().dict(), + database=database + ) + + @staticmethod + def __data_to_db(data: RSSItem) -> dict: + db_data = data.dict() + for key, value in db_data.items(): + if isinstance(value, bool): + db_data[key] = int(value) + elif isinstance(value, list): + db_data[key] = ",".join(value) + return db_data + + @staticmethod + def __db_to_data(db_data: dict) -> RSSItem: + for key, item in db_data.items(): + if isinstance(item, int): + db_data[key] = bool(item) + return RSSItem(**db_data) + + def update_table(self): + self.update.table() + + def insert_one(self, data: RSSItem): + dict_data = self.__data_to_db(data) + self.insert.one(data=dict_data) + + def get_all(self) -> list[RSSItem]: + dict_datas = self.select.all() + return [self.__db_to_data(x) for x in dict_datas] + + def delete_one(self, _id: int): + self.delete.one(_id) diff --git a/backend/src/module/rss/engine.py b/backend/src/module/rss/engine.py index 8befad0c..b0c668b9 100644 --- a/backend/src/module/rss/engine.py +++ b/backend/src/module/rss/engine.py @@ -6,7 +6,6 @@ from module.network import RequestContent, TorrentInfo class RSSEngine(RequestContent): - @staticmethod def _get_rss_items() -> list[RSSItem]: with RSSDatabase() as db: @@ -73,3 +72,8 @@ class RSSEngine(RequestContent): with TorrentDatabase() as db: db.insert_many(matched_torrents) return matched_torrents + + +if __name__ == '__main__': + with RSSEngine() as engine: + engine.run() From 4254651b631069ed6650a9208c56ce0738db8f5f Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Wed, 21 Jun 2023 15:48:24 +0800 Subject: [PATCH 3/9] temp: save --- .gitignore | 2 +- backend/src/module/database/bangumi.py | 16 ++++++++++- backend/src/module/database/orm/update.py | 34 ++++++++++++----------- backend/src/module/database/torrent.py | 15 +++++++--- backend/src/module/models/rss.py | 9 +++--- backend/src/module/rss/engine.py | 27 ++++++++---------- 6 files changed, 61 insertions(+), 42 deletions(-) diff --git a/.gitignore b/.gitignore index 1e5de80b..828f0945 100644 --- a/.gitignore +++ b/.gitignore @@ -178,6 +178,7 @@ test.* /backend/src/config/ /src/debuger.py /backend/src/dist.zip +/src # webui logs @@ -205,4 +206,3 @@ dist-ssr *.njsproj *.sln *.sw? - diff --git a/backend/src/module/database/bangumi.py b/backend/src/module/database/bangumi.py index 97efe1b4..c5983e89 100644 --- a/backend/src/module/database/bangumi.py +++ b/backend/src/module/database/bangumi.py @@ -169,7 +169,21 @@ class BangumiDatabase(Connector): dict_data = self.select.many(conditions=conditions, combine_operator="OR") return [self.__db_to_data(x) for x in dict_data] + def get_rss(self, rss_link: str) -> list[BangumiData]: + conditions = {"rss_link": rss_link} + dict_data = self.select.many(conditions=conditions, combine_operator="INSTR") + return [self.__db_to_data(x) for x in dict_data] + + def match_torrent(self, torrent_name: str, rss_link: str) -> BangumiData | None: + conditions = {"title_raw": torrent_name, "rss_link": rss_link} + dict_data = self.select.one(conditions=conditions, combine_operator="INSTR") + if not dict_data: + return None + return self.__db_to_data(dict_data) + if __name__ == "__main__": with BangumiDatabase() as db: - print(db.match_poster("久保")) + db.match_torrent( + "魔法科高校の劣等生 来訪者編", "https://bangumi.moe/rss/5f6b3e3e4e8c4b0001b2e3a3" + ) diff --git a/backend/src/module/database/orm/update.py b/backend/src/module/database/orm/update.py index 7b022418..000d5fe9 100644 --- a/backend/src/module/database/orm/update.py +++ b/backend/src/module/database/orm/update.py @@ -19,12 +19,11 @@ class Update: return self._connector.fetch() is not None def table(self): - columns = ", ".join( - [ - f"{key} {self.__python_to_sqlite_type(value)}" - for key, value in self._example_data.items() - ] - ) + columns_list = [ + self.__python_to_sqlite_type(key, value) + for key, value in self._example_data.items() + ] + columns = ", ".join(columns_list) create_table_sql = f"CREATE TABLE IF NOT EXISTS {self._table_name} ({columns});" self._connector.execute(create_table_sql) logger.debug(f"Create table {self._table_name}.") @@ -32,10 +31,10 @@ class Update: existing_columns = [x[1] for x in self._connector.fetch()] for key, value in self._example_data.items(): if key not in existing_columns: - insert_column = self.__python_to_sqlite_type(value) + insert_column = self.__python_to_sqlite_type(key, value) if value is None: value = "NULL" - add_column_sql = f"ALTER TABLE {self._table_name} ADD COLUMN {key} {insert_column} DEFAULT {value};" + add_column_sql = f"ALTER TABLE {self._table_name} ADD COLUMN {insert_column} DEFAULT {value};" self._connector.execute(add_column_sql) logger.debug(f"Update table {self._table_name}.") @@ -81,18 +80,21 @@ class Update: return True @staticmethod - def __python_to_sqlite_type(value) -> str: - if isinstance(value, int): - return "INTEGER NOT NULL" + def __python_to_sqlite_type(key, value) -> str: + if key == "id": + column = "INTEGER PRIMARY KEY" + elif isinstance(value, int): + column = "INTEGER NOT NULL" elif isinstance(value, float): - return "REAL NOT NULL" + column = "REAL NOT NULL" elif isinstance(value, str): - return "TEXT NOT NULL" + column = "TEXT NOT NULL" elif isinstance(value, bool): - return "INTEGER NOT NULL" + column = "INTEGER NOT NULL" elif isinstance(value, list): - return "TEXT NOT NULL" + column = "TEXT NOT NULL" elif value is None: - return "TEXT" + column = "TEXT" else: raise ValueError(f"Unsupported data type: {type(value)}") + return f"{key} {column}" diff --git a/backend/src/module/database/torrent.py b/backend/src/module/database/torrent.py index 2821182f..dd16823a 100644 --- a/backend/src/module/database/torrent.py +++ b/backend/src/module/database/torrent.py @@ -1,6 +1,6 @@ import logging -from .orm import Connector +from module.database.orm import Connector from module.models import TorrentData from module.conf import DATA_PATH @@ -10,9 +10,7 @@ logger = logging.getLogger(__name__) class TorrentDatabase(Connector): def __init__(self, database: str = DATA_PATH): super().__init__( - table_name="torrent", - data=TorrentData().dict(), - database=DATA_PATH + table_name="torrent", data=TorrentData().dict(), database=database ) def update_table(self): @@ -42,3 +40,12 @@ class TorrentDatabase(Connector): def get_all(self) -> list[TorrentData]: dict_datas = self.select.all() return [self.__db_to_data(data) for data in dict_datas] + + def get_torrent_name(self) -> list[str]: + dict_data = self.select.all() + return [data["name"] for data in dict_data] + + +if __name__ == "__main__": + with TorrentDatabase() as db: + db.update_table() diff --git a/backend/src/module/models/rss.py b/backend/src/module/models/rss.py index 66f38292..f9fe9aed 100644 --- a/backend/src/module/models/rss.py +++ b/backend/src/module/models/rss.py @@ -11,8 +11,7 @@ class RSSItem(BaseModel): class TorrentData(BaseModel): id: int = Field(0, alias="id") - name: str = Field(..., alias="name") - url: str = Field(..., alias="url") - matched: bool = Field(..., alias="matched") - downloaded: bool = Field(..., alias="downloaded") - save_path: str = Field(..., alias="save_path") + rss_id: int = Field(0, alias="rss_id") + name: str = Field("", alias="name") + url: str = Field("https://example.com/torrent", alias="url") + save_path: str = Field("path/to/save", alias="save_path") diff --git a/backend/src/module/rss/engine.py b/backend/src/module/rss/engine.py index b0c668b9..5fb7ff97 100644 --- a/backend/src/module/rss/engine.py +++ b/backend/src/module/rss/engine.py @@ -14,7 +14,7 @@ class RSSEngine(RequestContent): @staticmethod def _get_bangumi_data(rss_link: str) -> list[BangumiData]: with BangumiDatabase() as db: - return db.get_rss_data(rss_link) + return db.get_rss(rss_link) def add_rss(self, rss_link: str, name: str, combine: bool): if not name: @@ -28,9 +28,9 @@ class RSSEngine(RequestContent): return torrents @staticmethod - def match_torrent(torrent: TorrentInfo) -> TorrentData | None: + def match_torrent(torrent: TorrentInfo, rss_link: str) -> TorrentData | None: with BangumiDatabase() as db: - bangumi_data = db.match_torrent(torrent.name) + bangumi_data = db.match_torrent(torrent.name, rss_link) if bangumi_data: _filter = "|".join(bangumi_data.filter) if re.search(_filter, torrent.name): @@ -44,16 +44,13 @@ class RSSEngine(RequestContent): @staticmethod def filter_torrent(torrents: list[TorrentInfo]) -> list[TorrentInfo]: + new_torrents = [] with TorrentDatabase() as db: - in_db_torrents = db.get_all() - in_db_torrents = [x.name for x in in_db_torrents] - i = 0 - while i < len(torrents): - torrent = torrents[i] - if torrent.name in in_db_torrents: - torrents.pop(i) - i += 1 - return torrents + in_db_torrents: list = db.get_torrent_name() + for torrent in torrents: + if torrent.name not in in_db_torrents: + new_torrents.append(torrent) + return new_torrents def run(self): # Get All RSS Items @@ -61,10 +58,10 @@ class RSSEngine(RequestContent): # From RSS Items, get all torrents for rss_item in rss_items: torrents = self.get_torrents(rss_item.url) - self.filter_torrent(torrents) + new_torrents = self.filter_torrent(torrents) # Get all enabled bangumi data matched_torrents = [] - for torrent in torrents: + for torrent in new_torrents: matched_torrent = self.match_torrent(torrent) if matched_torrent: matched_torrents.append(matched_torrent) @@ -74,6 +71,6 @@ class RSSEngine(RequestContent): return matched_torrents -if __name__ == '__main__': +if __name__ == "__main__": with RSSEngine() as engine: engine.run() From 472a5093e956623f12d5ce1ed7600fb7fea609e4 Mon Sep 17 00:00:00 2001 From: estrella Date: Thu, 3 Aug 2023 14:16:52 +0800 Subject: [PATCH 4/9] refactor: database and rss engine --- backend/src/module/database/bangumi.py | 76 +++++++++--------- backend/src/module/database/combine.py | 18 +++++ backend/src/module/database/rss.py | 59 ++++++-------- backend/src/module/database/torrent.py | 77 ++++++++++--------- backend/src/module/models/__init__.py | 6 +- backend/src/module/models/bangumi.py | 4 + backend/src/module/models/config.py | 2 - backend/src/module/models/rss.py | 22 +++--- backend/src/module/models/torrent.py | 21 ++--- .../src/module/network/request_contents.py | 5 +- backend/src/module/rss/engine.py | 35 ++++++--- backend/src/test/test_database.py | 18 ++--- 12 files changed, 189 insertions(+), 154 deletions(-) create mode 100644 backend/src/module/database/combine.py diff --git a/backend/src/module/database/bangumi.py b/backend/src/module/database/bangumi.py index 9ab0b092..64de1994 100644 --- a/backend/src/module/database/bangumi.py +++ b/backend/src/module/database/bangumi.py @@ -4,35 +4,34 @@ from sqlmodel import Session, select, delete, or_ from sqlalchemy.sql import func from typing import Optional -from .engine import engine from module.models import Bangumi logger = logging.getLogger(__name__) -class BangumiDatabase(Session): - def __init__(self, _engine=engine): - super().__init__(_engine) +class BangumiDatabase: + def __init__(self, session: Session): + self.session = session def insert_one(self, data: Bangumi): - self.add(data) - self.commit() + self.session.add(data) + self.session.commit() logger.debug(f"[Database] Insert {data.official_title} into database.") def insert_list(self, data: list[Bangumi]): - self.add_all(data) + self.session.add_all(data) logger.debug(f"[Database] Insert {len(data)} bangumi into database.") def update_one(self, data: Bangumi) -> bool: - db_data = self.get(Bangumi, data.id) + db_data = self.session.get(Bangumi, data.id) if not db_data: return False bangumi_data = data.dict(exclude_unset=True) for key, value in bangumi_data.items(): setattr(db_data, key, value) - self.add(db_data) - self.commit() - self.refresh(db_data) + self.session.add(db_data) + self.session.commit() + self.session.refresh(db_data) logger.debug(f"[Database] Update {data.official_title}") return True @@ -43,53 +42,53 @@ class BangumiDatabase(Session): def update_rss(self, title_raw, rss_set: str): # Update rss and added statement = select(Bangumi).where(Bangumi.title_raw == title_raw) - bangumi = self.exec(statement).first() + bangumi = self.session.exec(statement).first() bangumi.rss_link = rss_set bangumi.added = False - self.add(bangumi) - self.commit() - self.refresh(bangumi) + self.session.add(bangumi) + self.session.commit() + self.session.refresh(bangumi) logger.debug(f"[Database] Update {title_raw} rss_link to {rss_set}.") def update_poster(self, title_raw, poster_link: str): statement = select(Bangumi).where(Bangumi.title_raw == title_raw) - bangumi = self.exec(statement).first() + bangumi = self.session.exec(statement).first() bangumi.poster_link = poster_link - self.add(bangumi) - self.commit() - self.refresh(bangumi) + self.session.add(bangumi) + self.session.commit() + self.session.refresh(bangumi) logger.debug(f"[Database] Update {title_raw} poster_link to {poster_link}.") def delete_one(self, _id: int): statement = select(Bangumi).where(Bangumi.id == _id) - bangumi = self.exec(statement).first() - self.delete(bangumi) - self.commit() + bangumi = self.session.exec(statement).first() + self.session.delete(bangumi) + self.session.commit() logger.debug(f"[Database] Delete bangumi id: {_id}.") def delete_all(self): statement = delete(Bangumi) - self.exec(statement) - self.commit() + self.session.exec(statement) + self.session.commit() def search_all(self) -> list[Bangumi]: statement = select(Bangumi) - return self.exec(statement).all() + return self.session.exec(statement).all() def search_id(self, _id: int) -> Optional[Bangumi]: statement = select(Bangumi).where(Bangumi.id == _id) - bangumi = self.exec(statement).first() + bangumi = self.session.exec(statement).first() if bangumi is None: logger.warning(f"[Database] Cannot find bangumi id: {_id}.") return None else: logger.debug(f"[Database] Find bangumi id: {_id}.") - return self.exec(statement).first() + return self.session.exec(statement).first() def match_poster(self, bangumi_name: str) -> str: # Use like to match statement = select(Bangumi).where(func.instr(bangumi_name, Bangumi.title_raw) > 0) - data = self.exec(statement).first() + data = self.session.exec(statement).first() if data: return data.poster_link else: @@ -119,7 +118,7 @@ class BangumiDatabase(Session): def not_complete(self) -> list[Bangumi]: # Find eps_complete = False condition = select(Bangumi).where(Bangumi.eps_collect == 0) - datas = self.exec(condition).all() + datas = self.session.exec(condition).all() return datas def not_added(self) -> list[Bangumi]: @@ -128,19 +127,20 @@ class BangumiDatabase(Session): Bangumi.added == 0, Bangumi.rule_name is None, Bangumi.save_path is None ) ) - datas = self.exec(conditions).all() + datas = self.session.exec(conditions).all() return datas def disable_rule(self, _id: int): statement = select(Bangumi).where(Bangumi.id == _id) - bangumi = self.exec(statement).first() + bangumi = self.session.exec(statement).first() bangumi.deleted = True - self.add(bangumi) - self.commit() - self.refresh(bangumi) + self.session.add(bangumi) + self.session.commit() + self.session.refresh(bangumi) logger.debug(f"[Database] Disable rule {bangumi.title_raw}.") - -if __name__ == "__main__": - with BangumiDatabase() as db: - print(db.not_complete()) + def search_rss(self, rss_link: str) -> list[Bangumi]: + statement = select(Bangumi).where( + func.instr(rss_link, Bangumi.rss_link) > 0 + ) + return self.session.exec(statement).all() \ No newline at end of file diff --git a/backend/src/module/database/combine.py b/backend/src/module/database/combine.py new file mode 100644 index 00000000..20c2e32e --- /dev/null +++ b/backend/src/module/database/combine.py @@ -0,0 +1,18 @@ +from sqlmodel import Session,SQLModel + +from .engine import engine +from .rss import RSSDatabase +from .torrent import TorrentDatabase +from .bangumi import BangumiDatabase + + +class Database(Session): + def __init__(self, _engine=engine): + super().__init__(_engine) + self.rss = RSSDatabase(self) + self.torrent = TorrentDatabase(self) + self.bangumi = BangumiDatabase(self) + + @staticmethod + def create_table(): + SQLModel.metadata.create_all(engine) \ No newline at end of file diff --git a/backend/src/module/database/rss.py b/backend/src/module/database/rss.py index 8c86ef66..72b08a0f 100644 --- a/backend/src/module/database/rss.py +++ b/backend/src/module/database/rss.py @@ -1,44 +1,33 @@ -from .orm import Connector +import logging +from sqlmodel import Session, select, delete + +from .engine import engine from module.models import RSSItem -from module.conf import DATA_PATH + +logger = logging.getLogger(__name__) -class RSSDatabase(Connector): - def __init__(self, database: str = DATA_PATH): - super().__init__( - table_name="RSSItem", - data=RSSItem().dict(), - database=database - ) - - @staticmethod - def __data_to_db(data: RSSItem) -> dict: - db_data = data.dict() - for key, value in db_data.items(): - if isinstance(value, bool): - db_data[key] = int(value) - elif isinstance(value, list): - db_data[key] = ",".join(value) - return db_data - - @staticmethod - def __db_to_data(db_data: dict) -> RSSItem: - for key, item in db_data.items(): - if isinstance(item, int): - db_data[key] = bool(item) - return RSSItem(**db_data) - - def update_table(self): - self.update.table() +class RSSDatabase: + def __init__(self, session: Session): + self.session = session def insert_one(self, data: RSSItem): - dict_data = self.__data_to_db(data) - self.insert.one(data=dict_data) + self.session.add(data) + self.session.commit() + self.session.refresh(data) - def get_all(self) -> list[RSSItem]: - dict_datas = self.select.all() - return [self.__db_to_data(x) for x in dict_datas] + def search_all(self) -> list[RSSItem]: + return self.session.exec(select(RSSItem)).all() def delete_one(self, _id: int): - self.delete.one(_id) + condition = delete(RSSItem).where(RSSItem.id == _id) + self.session.exec(condition) + self.session.commit() + + def delete_all(self): + condition = delete(RSSItem) + self.session.exec(condition) + self.session.commit() + + diff --git a/backend/src/module/database/torrent.py b/backend/src/module/database/torrent.py index dd16823a..e6fc2d02 100644 --- a/backend/src/module/database/torrent.py +++ b/backend/src/module/database/torrent.py @@ -1,51 +1,54 @@ import logging -from module.database.orm import Connector -from module.models import TorrentData -from module.conf import DATA_PATH +from sqlmodel import Session, select + +from module.models import Torrent logger = logging.getLogger(__name__) -class TorrentDatabase(Connector): - def __init__(self, database: str = DATA_PATH): - super().__init__( - table_name="torrent", data=TorrentData().dict(), database=database - ) +class TorrentDatabase: + def __init__(self, session: Session): + self.session = session - def update_table(self): - self.update.table() + def insert_one(self, data: Torrent): + self.session.add(data) + self.session.commit() + self.session.refresh(data) + logger.debug(f"Insert {data.name} in database.") - def __data_to_db(self, data: TorrentData) -> dict: - db_data = data.dict() - for key, value in db_data.items(): - if isinstance(value, bool): - db_data[key] = int(value) - elif isinstance(value, list): - db_data[key] = ",".join(value) - return db_data + def insert_many(self, datas: list[Torrent]): + self.session.add_all(datas) + self.session.commit() + logger.debug(f"Insert {len(datas)} torrents in database.") - def __db_to_data(self, db_data: dict) -> TorrentData: - for key, item in db_data.items(): - if isinstance(item, int): - db_data[key] = bool(item) - elif key in ["filter", "rss_link"]: - db_data[key] = item.split(",") - return TorrentData(**db_data) + def update_one_sys(self, data: Torrent): + self.session.add(data) + self.session.commit() + self.session.refresh(data) + logger.debug(f"Update {data.name} in database.") - def insert_many(self, data_list: list[TorrentData]): - dict_datas = [self.__data_to_db(data) for data in data_list] - self.insert.many(dict_datas) + def update_many_sys(self, datas: list[Torrent]): + self.session.add_all(datas) + self.session.commit() - def get_all(self) -> list[TorrentData]: - dict_datas = self.select.all() - return [self.__db_to_data(data) for data in dict_datas] + def update_one_user(self, data: Torrent): + self.session.add(data) + self.session.commit() + self.session.refresh(data) + logger.debug(f"Update {data.name} in database.") - def get_torrent_name(self) -> list[str]: - dict_data = self.select.all() - return [data["name"] for data in dict_data] + def search_one(self, _id: int) -> Torrent: + return self.session.exec(select(Torrent).where(Torrent.id == _id)).first() + def search_all(self) -> list[Torrent]: + return self.session.exec(select(Torrent)).all() -if __name__ == "__main__": - with TorrentDatabase() as db: - db.update_table() + def check_new(self, torrents_list: list[Torrent]) -> list[Torrent]: + new_torrents = [] + for torrent in torrents_list: + statement = select(Torrent).where(Torrent.name == torrent.name) + db_torrent = self.session.exec(statement).first() + if not db_torrent: + new_torrents.append(torrent) + return new_torrents diff --git a/backend/src/module/models/__init__.py b/backend/src/module/models/__init__.py index 0b10feaf..587d548f 100644 --- a/backend/src/module/models/__init__.py +++ b/backend/src/module/models/__init__.py @@ -1,5 +1,5 @@ -from .bangumi import Bangumi, Episode +from .bangumi import Bangumi, Episode, BangumiUpdate from .config import Config -from .rss import RSSTorrents -from .torrent import EpisodeFile, SubtitleFile, TorrentBase +from .rss import RSSItem, RSSUpdate +from .torrent import EpisodeFile, SubtitleFile, Torrent, TorrentUpdate from .user import UserLogin diff --git a/backend/src/module/models/bangumi.py b/backend/src/module/models/bangumi.py index b7484887..2613ff38 100644 --- a/backend/src/module/models/bangumi.py +++ b/backend/src/module/models/bangumi.py @@ -34,6 +34,7 @@ class BangumiUpdate(SQLModel): default="official_title", alias="official_title", title="番剧中文名" ) year: Optional[str] = Field(alias="year", title="番剧年份") + title_raw: str = Field(default="title_raw", alias="title_raw", title="番剧原名") season: int = Field(default=1, alias="season", title="番剧季度") season_raw: Optional[str] = Field(alias="season_raw", title="番剧季度原名") group_name: Optional[str] = Field(alias="group_name", title="字幕组") @@ -44,7 +45,10 @@ class BangumiUpdate(SQLModel): offset: int = Field(default=0, alias="offset", title="番剧偏移量") filter: str = Field(default="720, \\d+-\\d+", alias="filter", title="番剧过滤器") rss_link: str = Field(default="", alias="rss_link", title="番剧RSS链接") + poster_link: Optional[str] = Field(alias="poster_link", title="番剧海报链接") added: bool = Field(default=False, alias="added", title="是否已添加") + rule_name: Optional[str] = Field(alias="rule_name", title="番剧规则名") + save_path: Optional[str] = Field(alias="save_path", title="番剧保存路径") deleted: bool = Field(False, alias="deleted", title="是否已删除") diff --git a/backend/src/module/models/config.py b/backend/src/module/models/config.py index 76ec036f..a1169ad2 100644 --- a/backend/src/module/models/config.py +++ b/backend/src/module/models/config.py @@ -1,8 +1,6 @@ from os.path import expandvars from pydantic import BaseModel, Field -# Sub config - class Program(BaseModel): rss_time: int = Field(7200, description="Sleep time") diff --git a/backend/src/module/models/rss.py b/backend/src/module/models/rss.py index f9fe9aed..aa98f028 100644 --- a/backend/src/module/models/rss.py +++ b/backend/src/module/models/rss.py @@ -1,17 +1,21 @@ -from pydantic import BaseModel, Field +from sqlmodel import SQLModel, Field +from typing import Optional -class RSSItem(BaseModel): - id: int = Field(0, alias="id", title="id") +class RSSItem(SQLModel, table=True): + id: int = Field(default=None, primary_key=True, alias="id") item_path: str = Field("example path", alias="item_path") url: str = Field("https://mikanani.me", alias="url") combine: bool = Field(True, alias="combine") enabled: bool = Field(True, alias="enabled") -class TorrentData(BaseModel): - id: int = Field(0, alias="id") - rss_id: int = Field(0, alias="rss_id") - name: str = Field("", alias="name") - url: str = Field("https://example.com/torrent", alias="url") - save_path: str = Field("path/to/save", alias="save_path") +class RSSUpdate(SQLModel): + item_path: Optional[str] = Field("example path", alias="item_path") + url: Optional[str] = Field("https://mikanani.me", alias="url") + combine: Optional[bool] = Field(True, alias="combine") + enabled: Optional[bool] = Field(True, alias="enabled") + + + + diff --git a/backend/src/module/models/torrent.py b/backend/src/module/models/torrent.py index 892d66d6..e44f6104 100644 --- a/backend/src/module/models/torrent.py +++ b/backend/src/module/models/torrent.py @@ -1,16 +1,19 @@ -from pydantic import BaseModel, Field +from pydantic import BaseModel +from sqlmodel import SQLModel, Field +from typing import Optional -class TorrentBase(BaseModel): - name: str = Field(...) - torrent_link: str = Field(...) - homepage: str | None = Field(None) +class Torrent(SQLModel, table=True): + id: int = Field(default=None, primary_key=True, alias="id") + refer_id: Optional[int] = Field(None, alias="refer_id") + name: str = Field("", alias="name") + url: str = Field("https://example.com/torrent", alias="url") + homepage: Optional[str] = Field(None, alias="homepage") + downloaded: bool = Field(False, alias="downloaded") -class FileSet(BaseModel): - media_path: str = Field(...) - sc_subtitle: str | None = Field(None) - tc_subtitle: str | None = Field(None) +class TorrentUpdate(SQLModel): + downloaded: bool = Field(False, alias="downloaded") class EpisodeFile(BaseModel): diff --git a/backend/src/module/network/request_contents.py b/backend/src/module/network/request_contents.py index 7f00a1d1..c42018f0 100644 --- a/backend/src/module/network/request_contents.py +++ b/backend/src/module/network/request_contents.py @@ -37,7 +37,6 @@ class TorrentInfo: class RequestContent(RequestURL): - # Mikanani RSS def get_torrents( self, _url: str, @@ -103,4 +102,6 @@ class RequestContent(RequestURL): if __name__ == '__main__': with RequestContent() as req: - req.get_xml("https://mikanani.me/RSS/Classic") + ts = req.get_torrents("https://mikanani.me/RSS/Classic") + for t in ts: + print(t) diff --git a/backend/src/module/rss/engine.py b/backend/src/module/rss/engine.py index 5fb7ff97..da68c701 100644 --- a/backend/src/module/rss/engine.py +++ b/backend/src/module/rss/engine.py @@ -1,20 +1,35 @@ import re from module.database import RSSDatabase, BangumiDatabase, TorrentDatabase -from module.models import BangumiData, RSSItem, TorrentData +from module.models import Bangumi, RSSItem, Torrent from module.network import RequestContent, TorrentInfo +from module.database.combine import Database -class RSSEngine(RequestContent): - @staticmethod - def _get_rss_items() -> list[RSSItem]: - with RSSDatabase() as db: - return db.get_all() - @staticmethod - def _get_bangumi_data(rss_link: str) -> list[BangumiData]: - with BangumiDatabase() as db: - return db.get_rss(rss_link) +class RSSEngine(Database): + def _get_rss_items(self) -> list[RSSItem]: + return self.rss.search_all() + + def _get_bangumi_data(self, rss_link: str) -> list[Bangumi]: + return self.bangumi.search_rss(rss_link) + + def get_torrent(self, rss_link: str) -> list[Torrent]: + with RequestContent() as req: + torrent_infos = req.get_torrents(rss_link) + torrents: list[Torrent] = [] + for torrent_info in torrent_infos: + torrents.append( + Torrent( + name=torrent_info.name, + url=torrent_info.torrent_link, + homepage=torrent_info.homepage, + ) + ) + return torrents + + def check_new_torrents(self, torrents_list: list[list[Torrent]]) -> list[Torrent]: + return self.torrent.check_new(torrents_list) def add_rss(self, rss_link: str, name: str, combine: bool): if not name: diff --git a/backend/src/test/test_database.py b/backend/src/test/test_database.py index 91f58f14..4f3e983b 100644 --- a/backend/src/test/test_database.py +++ b/backend/src/test/test_database.py @@ -1,7 +1,7 @@ from sqlmodel import create_engine, SQLModel from sqlmodel.pool import StaticPool -from module.database import BangumiDatabase +from module.database.combine import Database from module.models import Bangumi @@ -31,19 +31,19 @@ def test_bangumi_database(): save_path=None, deleted=False, ) - with BangumiDatabase(engine) as database: + with Database(engine) as db: # insert - database.insert_one(test_data) - assert database.search_id(1) == test_data + db.bangumi.insert_one(test_data) + assert db.bangumi.search_id(1) == test_data # update test_data.official_title = "test2" - database.update_one(test_data) - assert database.search_id(1) == test_data + db.bangumi.update_one(test_data) + assert db.bangumi.search_id(1) == test_data # search poster - assert database.match_poster("test2 (2021)") == "/test/test.jpg" + assert db.bangumi.match_poster("test2 (2021)") == "/test/test.jpg" # delete - database.delete_one(1) - assert database.search_id(1) is None + db.bangumi.delete_one(1) + assert db.bangumi.search_id(1) is None From 80ea9ec65704e9b8e22de01c51559203a118a72b Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Fri, 4 Aug 2023 19:02:29 +0800 Subject: [PATCH 5/9] feat: Complete RSS engine. --- backend/src/module/database/bangumi.py | 18 ++-- backend/src/module/database/combine.py | 4 +- backend/src/module/database/rss.py | 22 ++++- backend/src/module/database/torrent.py | 10 +-- backend/src/module/models/torrent.py | 1 + .../src/module/network/request_contents.py | 9 +- backend/src/module/rss/engine.py | 83 +++++++------------ 7 files changed, 68 insertions(+), 79 deletions(-) diff --git a/backend/src/module/database/bangumi.py b/backend/src/module/database/bangumi.py index 64de1994..a432dc94 100644 --- a/backend/src/module/database/bangumi.py +++ b/backend/src/module/database/bangumi.py @@ -1,6 +1,6 @@ import logging -from sqlmodel import Session, select, delete, or_ +from sqlmodel import Session, select, delete, or_, and_ from sqlalchemy.sql import func from typing import Optional @@ -87,7 +87,9 @@ class BangumiDatabase: def match_poster(self, bangumi_name: str) -> str: # Use like to match - statement = select(Bangumi).where(func.instr(bangumi_name, Bangumi.title_raw) > 0) + statement = select(Bangumi).where( + func.instr(bangumi_name, Bangumi.title_raw) > 0 + ) data = self.session.exec(statement).first() if data: return data.poster_link @@ -115,6 +117,12 @@ class BangumiDatabase: i += 1 return torrent_list + def match_torrent(self, torrent_name: str) -> Optional[Bangumi]: + statement = select(Bangumi).where( + and_(func.instr(torrent_name, Bangumi.title_raw) > 0, not Bangumi.deleted) + ) + return self.session.exec(statement).first() + def not_complete(self) -> list[Bangumi]: # Find eps_complete = False condition = select(Bangumi).where(Bangumi.eps_collect == 0) @@ -140,7 +148,5 @@ class BangumiDatabase: logger.debug(f"[Database] Disable rule {bangumi.title_raw}.") def search_rss(self, rss_link: str) -> list[Bangumi]: - statement = select(Bangumi).where( - func.instr(rss_link, Bangumi.rss_link) > 0 - ) - return self.session.exec(statement).all() \ No newline at end of file + statement = select(Bangumi).where(func.instr(rss_link, Bangumi.rss_link) > 0) + return self.session.exec(statement).all() diff --git a/backend/src/module/database/combine.py b/backend/src/module/database/combine.py index 20c2e32e..9066f378 100644 --- a/backend/src/module/database/combine.py +++ b/backend/src/module/database/combine.py @@ -1,4 +1,4 @@ -from sqlmodel import Session,SQLModel +from sqlmodel import Session, SQLModel from .engine import engine from .rss import RSSDatabase @@ -15,4 +15,4 @@ class Database(Session): @staticmethod def create_table(): - SQLModel.metadata.create_all(engine) \ No newline at end of file + SQLModel.metadata.create_all(engine) diff --git a/backend/src/module/database/rss.py b/backend/src/module/database/rss.py index 72b08a0f..bc539280 100644 --- a/backend/src/module/database/rss.py +++ b/backend/src/module/database/rss.py @@ -12,7 +12,20 @@ class RSSDatabase: def __init__(self, session: Session): self.session = session - def insert_one(self, data: RSSItem): + def add(self, data: RSSItem): + # Check if exists + statement = select(RSSItem).where(RSSItem.url == data.url) + db_data = self.session.exec(statement).first() + if db_data: + logger.debug(f"RSS Item {data.url} already exists.") + return + else: + logger.debug(f"RSS Item {data.url} not exists, adding...") + self.session.add(data) + self.session.commit() + self.session.refresh(data) + + def update(self, data: RSSItem): self.session.add(data) self.session.commit() self.session.refresh(data) @@ -20,7 +33,10 @@ class RSSDatabase: def search_all(self) -> list[RSSItem]: return self.session.exec(select(RSSItem)).all() - def delete_one(self, _id: int): + def search_active(self) -> list[RSSItem]: + return self.session.exec(select(RSSItem).where(RSSItem.enabled)).all() + + def delete(self, _id: int): condition = delete(RSSItem).where(RSSItem.id == _id) self.session.exec(condition) self.session.commit() @@ -29,5 +45,3 @@ class RSSDatabase: condition = delete(RSSItem) self.session.exec(condition) self.session.commit() - - diff --git a/backend/src/module/database/torrent.py b/backend/src/module/database/torrent.py index e6fc2d02..2c32dfca 100644 --- a/backend/src/module/database/torrent.py +++ b/backend/src/module/database/torrent.py @@ -11,13 +11,13 @@ class TorrentDatabase: def __init__(self, session: Session): self.session = session - def insert_one(self, data: Torrent): + def add(self, data: Torrent): self.session.add(data) self.session.commit() self.session.refresh(data) logger.debug(f"Insert {data.name} in database.") - def insert_many(self, datas: list[Torrent]): + def add_all(self, datas: list[Torrent]): self.session.add_all(datas) self.session.commit() logger.debug(f"Insert {len(datas)} torrents in database.") @@ -46,9 +46,9 @@ class TorrentDatabase: def check_new(self, torrents_list: list[Torrent]) -> list[Torrent]: new_torrents = [] + old_torrents = self.search_all() + old_urls = [t.url for t in old_torrents] for torrent in torrents_list: - statement = select(Torrent).where(Torrent.name == torrent.name) - db_torrent = self.session.exec(statement).first() - if not db_torrent: + if torrent.url not in old_urls: new_torrents.append(torrent) return new_torrents diff --git a/backend/src/module/models/torrent.py b/backend/src/module/models/torrent.py index e44f6104..1e6198d7 100644 --- a/backend/src/module/models/torrent.py +++ b/backend/src/module/models/torrent.py @@ -9,6 +9,7 @@ class Torrent(SQLModel, table=True): name: str = Field("", alias="name") url: str = Field("https://example.com/torrent", alias="url") homepage: Optional[str] = Field(None, alias="homepage") + save_path: Optional[str] = Field(None, alias="saved_path") downloaded: bool = Field(False, alias="downloaded") diff --git a/backend/src/module/network/request_contents.py b/backend/src/module/network/request_contents.py index c42018f0..c2e865fa 100644 --- a/backend/src/module/network/request_contents.py +++ b/backend/src/module/network/request_contents.py @@ -97,11 +97,4 @@ class RequestContent(RequestURL): def get_rss_title(self, _url): soup = self.get_xml(_url) - return soup.find("title").text - - -if __name__ == '__main__': - with RequestContent() as req: - ts = req.get_torrents("https://mikanani.me/RSS/Classic") - for t in ts: - print(t) + return soup.find("./channel/title").text diff --git a/backend/src/module/rss/engine.py b/backend/src/module/rss/engine.py index da68c701..1c775ce7 100644 --- a/backend/src/module/rss/engine.py +++ b/backend/src/module/rss/engine.py @@ -1,20 +1,16 @@ import re +import logging -from module.database import RSSDatabase, BangumiDatabase, TorrentDatabase from module.models import Bangumi, RSSItem, Torrent from module.network import RequestContent, TorrentInfo +from module.downloader import DownloadClient from module.database.combine import Database class RSSEngine(Database): - def _get_rss_items(self) -> list[RSSItem]: - return self.rss.search_all() - - def _get_bangumi_data(self, rss_link: str) -> list[Bangumi]: - return self.bangumi.search_rss(rss_link) - - def get_torrent(self, rss_link: str) -> list[Torrent]: + @staticmethod + def _get_torrents(rss_link: str) -> list[Torrent]: with RequestContent() as req: torrent_infos = req.get_torrents(rss_link) torrents: list[Torrent] = [] @@ -28,62 +24,41 @@ class RSSEngine(Database): ) return torrents - def check_new_torrents(self, torrents_list: list[list[Torrent]]) -> list[Torrent]: - return self.torrent.check_new(torrents_list) - - def add_rss(self, rss_link: str, name: str, combine: bool): + def add_rss(self, rss_link: str, name: str | None = None, combine: bool = True): if not name: - name = self.get_rss_title(rss_link) - insert_data = RSSItem(item_path=name, url=rss_link, combine=combine) - with RSSDatabase() as db: - db.insert_one(insert_data) + with RequestContent() as req: + name = req.get_rss_title(rss_link) + rss_data = RSSItem(item_path=name, url=rss_link, combine=combine) + self.rss.add(rss_data) - def pull_rss(self, rss_item: RSSItem) -> list[TorrentInfo]: - torrents = self.get_torrents(rss_item.url) - return torrents - - @staticmethod - def match_torrent(torrent: TorrentInfo, rss_link: str) -> TorrentData | None: - with BangumiDatabase() as db: - bangumi_data = db.match_torrent(torrent.name, rss_link) - if bangumi_data: - _filter = "|".join(bangumi_data.filter) - if re.search(_filter, torrent.name): - return None - else: - return TorrentData( - name=torrent.name, - url=torrent.torrent_link, - ) - return None - - @staticmethod - def filter_torrent(torrents: list[TorrentInfo]) -> list[TorrentInfo]: - new_torrents = [] - with TorrentDatabase() as db: - in_db_torrents: list = db.get_torrent_name() - for torrent in torrents: - if torrent.name not in in_db_torrents: - new_torrents.append(torrent) + def pull_rss(self, rss_item: RSSItem) -> list[Torrent]: + torrents = self._get_torrents(rss_item.url) + new_torrents = self.torrent.check_new(torrents) return new_torrents + def match_torrent(self, torrent: Torrent): + matched: Bangumi = self.bangumi.match_torrent(torrent.name) + if matched: + torrent.refer_id = matched.id + torrent.save_path = matched.save_path + with RequestContent() as req: + torrent_file = req.get_content(torrent.url) + with DownloadClient() as client: + client.add_torrent( + {"torrent_files": torrent_file, "save_path": torrent.save_path} + ) + torrent.downloaded = True + def run(self): # Get All RSS Items - rss_items: list[RSSItem] = self._get_rss_items() + rss_items: list[RSSItem] = self.rss.search_active() # From RSS Items, get all torrents for rss_item in rss_items: - torrents = self.get_torrents(rss_item.url) - new_torrents = self.filter_torrent(torrents) + new_torrents = self.pull_rss(rss_item) # Get all enabled bangumi data - matched_torrents = [] for torrent in new_torrents: - matched_torrent = self.match_torrent(torrent) - if matched_torrent: - matched_torrents.append(matched_torrent) - # Add to database - with TorrentDatabase() as db: - db.insert_many(matched_torrents) - return matched_torrents + self.match_torrent(torrent) + self.torrent.add_all(new_torrents) if __name__ == "__main__": From f99056c8d6010afea99fc24bc7bbe4c8093f3623 Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Sat, 5 Aug 2023 20:19:06 +0800 Subject: [PATCH 6/9] feat: add test for database and rss engine. --- backend/src/module/database/__init__.py | 5 ++- backend/src/module/database/bangumi.py | 20 ++++++----- backend/src/module/database/combine.py | 11 +++--- backend/src/module/database/torrent.py | 6 ++-- backend/src/module/rss/analyser.py | 8 ++--- backend/src/module/rss/engine.py | 9 +++-- backend/src/test/test_database.py | 45 ++++++++++++++++++++----- backend/src/test/test_rss_engine.py | 22 ++++++++++++ 8 files changed, 91 insertions(+), 35 deletions(-) create mode 100644 backend/src/test/test_rss_engine.py diff --git a/backend/src/module/database/__init__.py b/backend/src/module/database/__init__.py index cd90685d..87d8425a 100644 --- a/backend/src/module/database/__init__.py +++ b/backend/src/module/database/__init__.py @@ -1,3 +1,2 @@ -from .bangumi import BangumiDatabase -from .rss import RSSDatabase -from .torrent import TorrentDatabase \ No newline at end of file +from .combine import Database +from .engine import engine diff --git a/backend/src/module/database/bangumi.py b/backend/src/module/database/bangumi.py index a432dc94..29273102 100644 --- a/backend/src/module/database/bangumi.py +++ b/backend/src/module/database/bangumi.py @@ -13,16 +13,16 @@ class BangumiDatabase: def __init__(self, session: Session): self.session = session - def insert_one(self, data: Bangumi): + def add(self, data: Bangumi): self.session.add(data) self.session.commit() logger.debug(f"[Database] Insert {data.official_title} into database.") - def insert_list(self, data: list[Bangumi]): + def add_all(self, data: list[Bangumi]): self.session.add_all(data) logger.debug(f"[Database] Insert {len(data)} bangumi into database.") - def update_one(self, data: Bangumi) -> bool: + def update(self, data: Bangumi) -> bool: db_data = self.session.get(Bangumi, data.id) if not db_data: return False @@ -35,9 +35,10 @@ class BangumiDatabase: logger.debug(f"[Database] Update {data.official_title}") return True - def update_list(self, datas: list[Bangumi]): - for data in datas: - self.update_one(data) + def update_all(self, datas: list[Bangumi]): + self.session.add_all(datas) + self.session.commit() + logger.debug(f"[Database] Update {len(datas)} bangumi.") def update_rss(self, title_raw, rss_set: str): # Update rss and added @@ -119,13 +120,16 @@ class BangumiDatabase: def match_torrent(self, torrent_name: str) -> Optional[Bangumi]: statement = select(Bangumi).where( - and_(func.instr(torrent_name, Bangumi.title_raw) > 0, not Bangumi.deleted) + and_( + func.instr(torrent_name, Bangumi.title_raw) > 0, + Bangumi.deleted == False, + ) ) return self.session.exec(statement).first() def not_complete(self) -> list[Bangumi]: # Find eps_complete = False - condition = select(Bangumi).where(Bangumi.eps_collect == 0) + condition = select(Bangumi).where(Bangumi.eps_collect == False) datas = self.session.exec(condition).all() return datas diff --git a/backend/src/module/database/combine.py b/backend/src/module/database/combine.py index 9066f378..e6dce3a7 100644 --- a/backend/src/module/database/combine.py +++ b/backend/src/module/database/combine.py @@ -1,18 +1,17 @@ from sqlmodel import Session, SQLModel -from .engine import engine from .rss import RSSDatabase from .torrent import TorrentDatabase from .bangumi import BangumiDatabase class Database(Session): - def __init__(self, _engine=engine): - super().__init__(_engine) + def __init__(self, engine): + self.engine = engine + super().__init__(engine) self.rss = RSSDatabase(self) self.torrent = TorrentDatabase(self) self.bangumi = BangumiDatabase(self) - @staticmethod - def create_table(): - SQLModel.metadata.create_all(engine) + def create_table(self): + SQLModel.metadata.create_all(self.engine) diff --git a/backend/src/module/database/torrent.py b/backend/src/module/database/torrent.py index 2c32dfca..3ccafb14 100644 --- a/backend/src/module/database/torrent.py +++ b/backend/src/module/database/torrent.py @@ -22,13 +22,13 @@ class TorrentDatabase: self.session.commit() logger.debug(f"Insert {len(datas)} torrents in database.") - def update_one_sys(self, data: Torrent): + def update(self, data: Torrent): self.session.add(data) self.session.commit() self.session.refresh(data) logger.debug(f"Update {data.name} in database.") - def update_many_sys(self, datas: list[Torrent]): + def update_all(self, datas: list[Torrent]): self.session.add_all(datas) self.session.commit() @@ -38,7 +38,7 @@ class TorrentDatabase: self.session.refresh(data) logger.debug(f"Update {data.name} in database.") - def search_one(self, _id: int) -> Torrent: + def search(self, _id: int) -> Torrent: return self.session.exec(select(Torrent).where(Torrent.id == _id)).first() def search_all(self) -> list[Torrent]: diff --git a/backend/src/module/rss/analyser.py b/backend/src/module/rss/analyser.py index 00bde7a6..9746547d 100644 --- a/backend/src/module/rss/analyser.py +++ b/backend/src/module/rss/analyser.py @@ -2,8 +2,8 @@ import logging import re from module.conf import settings -from module.database import BangumiDatabase from module.models import Bangumi +from module.database import Database from module.network import RequestContent, TorrentInfo from module.parser import TitleParser @@ -13,8 +13,6 @@ logger = logging.getLogger(__name__) class RSSAnalyser: def __init__(self): self._title_analyser = TitleParser() - with BangumiDatabase() as db: - db.update_table() def official_title_parser(self, data: Bangumi, mikan_title: str): if settings.rss_parser.parser_type == "mikan": @@ -78,10 +76,10 @@ class RSSAnalyser: return data def rss_to_data( - self, rss_link: str, database: BangumiDatabase, full_parse: bool = True + self, rss_link: str, database: Database, full_parse: bool = True ) -> list[Bangumi]: rss_torrents = self.get_rss_torrents(rss_link, full_parse) - torrents_to_add = database.match_list(rss_torrents, rss_link) + torrents_to_add = database.bangumi.match_list(rss_torrents, rss_link) if not torrents_to_add: logger.debug("[RSS] No new title has been found.") return [] diff --git a/backend/src/module/rss/engine.py b/backend/src/module/rss/engine.py index 1c775ce7..5b608798 100644 --- a/backend/src/module/rss/engine.py +++ b/backend/src/module/rss/engine.py @@ -2,13 +2,18 @@ import re import logging from module.models import Bangumi, RSSItem, Torrent -from module.network import RequestContent, TorrentInfo +from module.network import RequestContent from module.downloader import DownloadClient -from module.database.combine import Database +from module.database import Database, engine + +logger = logging.getLogger(__name__) class RSSEngine(Database): + def __init__(self, _engine=engine): + super().__init__(_engine) + @staticmethod def _get_torrents(rss_link: str) -> list[Torrent]: with RequestContent() as req: diff --git a/backend/src/test/test_database.py b/backend/src/test/test_database.py index 4f3e983b..cfe9289e 100644 --- a/backend/src/test/test_database.py +++ b/backend/src/test/test_database.py @@ -2,15 +2,16 @@ from sqlmodel import create_engine, SQLModel from sqlmodel.pool import StaticPool from module.database.combine import Database -from module.models import Bangumi +from module.models import Bangumi, Torrent, RSSItem + + +# sqlite mock engine +engine = create_engine( + "sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool +) def test_bangumi_database(): - # sqlite mock engine - engine = create_engine( - "sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool - ) - SQLModel.metadata.create_all(engine) test_data = Bangumi( official_title="test", year="2021", @@ -32,18 +33,46 @@ def test_bangumi_database(): deleted=False, ) with Database(engine) as db: + db.create_table() # insert - db.bangumi.insert_one(test_data) + db.bangumi.add(test_data) assert db.bangumi.search_id(1) == test_data # update test_data.official_title = "test2" - db.bangumi.update_one(test_data) + db.bangumi.update(test_data) assert db.bangumi.search_id(1) == test_data # search poster assert db.bangumi.match_poster("test2 (2021)") == "/test/test.jpg" + # match torrent + result = db.bangumi.match_torrent("[Sub Group]test S02 01 [720p].mkv") + assert result.official_title == "test2" + # delete db.bangumi.delete_one(1) assert db.bangumi.search_id(1) is None + + +def test_torrent_database(): + test_data = Torrent( + name="[Sub Group]test S02 01 [720p].mkv", + url="https://test.com/test.mkv", + ) + with Database(engine) as db: + # insert + db.torrent.add(test_data) + assert db.torrent.search(1) == test_data + + # update + test_data.downloaded = True + db.torrent.update(test_data) + assert db.torrent.search(1) == test_data + + +def test_rss_database(): + rss_url = "https://test.com/test.xml" + + with Database(engine) as db: + db.rss.add(RSSItem(url=rss_url)) diff --git a/backend/src/test/test_rss_engine.py b/backend/src/test/test_rss_engine.py new file mode 100644 index 00000000..e2e8f90f --- /dev/null +++ b/backend/src/test/test_rss_engine.py @@ -0,0 +1,22 @@ +from module.rss.engine import RSSEngine + +from .test_database import engine as e + + +def test_rss_engine(): + with RSSEngine(e) as engine: + rss_link = "https://mikanani.me/RSS/Bangumi?bangumiId=2353&subgroupid=552" + + engine.add_rss(rss_link, combine=False) + + result = engine.rss.search_active() + + assert result[1].item_path == "Mikan Project - 无职转生～到了异世界就拿出真本事～" + + new_torrents = engine.pull_rss(result[1]) + torrent = new_torrents[0] + + assert torrent.name == "[Lilith-Raws] 无职转生，到了异世界就拿出真本事 / Mushoku Tensei - 11 [Baha][WEB-DL][1080p][AVC AAC][CHT][MP4]" + + + From 94579d6b1208c396f87f254afd56a48b34b77857 Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Sun, 6 Aug 2023 01:34:59 +0800 Subject: [PATCH 7/9] fix: engine bugs --- backend/src/module/database/bangumi.py | 2 +- backend/src/module/models/bangumi.py | 2 +- backend/src/module/rss/engine.py | 26 +++++++++++++++----------- backend/src/test/test_database.py | 24 ++++++++++++------------ backend/src/test/test_rss_engine.py | 4 ---- 5 files changed, 29 insertions(+), 29 deletions(-) diff --git a/backend/src/module/database/bangumi.py b/backend/src/module/database/bangumi.py index 29273102..017b3b5f 100644 --- a/backend/src/module/database/bangumi.py +++ b/backend/src/module/database/bangumi.py @@ -89,7 +89,7 @@ class BangumiDatabase: def match_poster(self, bangumi_name: str) -> str: # Use like to match statement = select(Bangumi).where( - func.instr(bangumi_name, Bangumi.title_raw) > 0 + func.instr(bangumi_name, Bangumi.official_title) > 0 ) data = self.session.exec(statement).first() if data: diff --git a/backend/src/module/models/bangumi.py b/backend/src/module/models/bangumi.py index 2613ff38..a4984111 100644 --- a/backend/src/module/models/bangumi.py +++ b/backend/src/module/models/bangumi.py @@ -20,7 +20,7 @@ class Bangumi(SQLModel, table=True): subtitle: Optional[str] = Field(alias="subtitle", title="字幕") eps_collect: bool = Field(default=False, alias="eps_collect", title="是否已收集") offset: int = Field(default=0, alias="offset", title="番剧偏移量") - filter: str = Field(default="720, \\d+-\\d+", alias="filter", title="番剧过滤器") + filter: str = Field(default="720,\\d+-\\d+", alias="filter", title="番剧过滤器") rss_link: str = Field(default="", alias="rss_link", title="番剧RSS链接") poster_link: Optional[str] = Field(alias="poster_link", title="番剧海报链接") added: bool = Field(default=False, alias="added", title="是否已添加") diff --git a/backend/src/module/rss/engine.py b/backend/src/module/rss/engine.py index 5b608798..7ca5524e 100644 --- a/backend/src/module/rss/engine.py +++ b/backend/src/module/rss/engine.py @@ -44,17 +44,18 @@ class RSSEngine(Database): def match_torrent(self, torrent: Torrent): matched: Bangumi = self.bangumi.match_torrent(torrent.name) if matched: - torrent.refer_id = matched.id - torrent.save_path = matched.save_path - with RequestContent() as req: - torrent_file = req.get_content(torrent.url) - with DownloadClient() as client: - client.add_torrent( - {"torrent_files": torrent_file, "save_path": torrent.save_path} - ) - torrent.downloaded = True + _filter = matched.filter.replace(",", "|") + if re.search(_filter, torrent.name, re.IGNORECASE): + torrent.refer_id = matched.id + torrent.save_path = matched.save_path + with RequestContent() as req: + torrent_file = req.get_content(torrent.url) + return { + "torrent_files": torrent_file, + "save_path": torrent.save_path, + } - def run(self): + def run(self, client: DownloadClient): # Get All RSS Items rss_items: list[RSSItem] = self.rss.search_active() # From RSS Items, get all torrents @@ -62,7 +63,10 @@ class RSSEngine(Database): new_torrents = self.pull_rss(rss_item) # Get all enabled bangumi data for torrent in new_torrents: - self.match_torrent(torrent) + download_info = self.match_torrent(torrent) + client.add_torrent(download_info) + torrent.downloaded = True + # Add all torrents to database self.torrent.add_all(new_torrents) diff --git a/backend/src/test/test_database.py b/backend/src/test/test_database.py index cfe9289e..bb2babf7 100644 --- a/backend/src/test/test_database.py +++ b/backend/src/test/test_database.py @@ -13,15 +13,15 @@ engine = create_engine( def test_bangumi_database(): test_data = Bangumi( - official_title="test", + official_title="无职转生，到了异世界就拿出真本事", year="2021", - title_raw="test", + title_raw="Mushoku Tensei", season=1, - season_raw="第一季", - group_name="test", - dpi="720p", - source="test", - subtitle="test", + season_raw="", + group_name="Lilith-Raws", + dpi="1080p", + source="Baha", + subtitle="CHT", eps_collect=False, offset=0, filter="720p,\\d+-\\d+", @@ -29,7 +29,7 @@ def test_bangumi_database(): poster_link="/test/test.jpg", added=False, rule_name=None, - save_path=None, + save_path="downloads/无职转生，到了异世界就拿出真本事/Season 1", deleted=False, ) with Database(engine) as db: @@ -39,16 +39,16 @@ def test_bangumi_database(): assert db.bangumi.search_id(1) == test_data # update - test_data.official_title = "test2" + test_data.official_title = "无职转生，到了异世界就拿出真本事II" db.bangumi.update(test_data) assert db.bangumi.search_id(1) == test_data # search poster - assert db.bangumi.match_poster("test2 (2021)") == "/test/test.jpg" + assert db.bangumi.match_poster("无职转生，到了异世界就拿出真本事II (2021)") == "/test/test.jpg" # match torrent - result = db.bangumi.match_torrent("[Sub Group]test S02 01 [720p].mkv") - assert result.official_title == "test2" + result = db.bangumi.match_torrent("[Lilith-Raws] 无职转生，到了异世界就拿出真本事 / Mushoku Tensei - 11 [Baha][WEB-DL][1080p][AVC AAC][CHT][MP4]") + assert result.official_title == "无职转生，到了异世界就拿出真本事II" # delete db.bangumi.delete_one(1) diff --git a/backend/src/test/test_rss_engine.py b/backend/src/test/test_rss_engine.py index e2e8f90f..4373f310 100644 --- a/backend/src/test/test_rss_engine.py +++ b/backend/src/test/test_rss_engine.py @@ -10,13 +10,9 @@ def test_rss_engine(): engine.add_rss(rss_link, combine=False) result = engine.rss.search_active() - assert result[1].item_path == "Mikan Project - 无职转生～到了异世界就拿出真本事～" new_torrents = engine.pull_rss(result[1]) torrent = new_torrents[0] - assert torrent.name == "[Lilith-Raws] 无职转生，到了异世界就拿出真本事 / Mushoku Tensei - 11 [Baha][WEB-DL][1080p][AVC AAC][CHT][MP4]" - - From d768299c7f44baaf3f1d420fb536a0929cf87d7e Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Sun, 6 Aug 2023 16:04:38 +0800 Subject: [PATCH 8/9] fix: fix bugs. --- backend/src/module/core/sub_thread.py | 22 ++++---- backend/src/module/database/combine.py | 3 +- backend/src/module/database/rss.py | 3 ++ .../module/downloader/client/qb_downloader.py | 3 +- .../src/module/downloader/download_client.py | 23 +++++--- backend/src/module/manager/collector.py | 53 +++++++------------ backend/src/module/manager/torrent.py | 24 ++++----- backend/src/module/models/__init__.py | 2 +- .../src/module/network/request_contents.py | 6 +-- .../src/module/notification/notification.py | 6 +-- backend/src/module/parser/title_parser.py | 3 +- backend/src/module/rss/__init__.py | 3 +- backend/src/module/rss/analyser.py | 29 +++++----- backend/src/module/rss/engine.py | 31 +++++------ backend/src/module/searcher/searcher.py | 8 +-- backend/src/module/update/data_migration.py | 8 +-- 16 files changed, 108 insertions(+), 119 deletions(-) diff --git a/backend/src/module/core/sub_thread.py b/backend/src/module/core/sub_thread.py index 26eaa676..15d9f781 100644 --- a/backend/src/module/core/sub_thread.py +++ b/backend/src/module/core/sub_thread.py @@ -2,11 +2,11 @@ import threading import time from module.conf import settings -from module.database import BangumiDatabase +from module.database import Database from module.downloader import DownloadClient from module.manager import Renamer, eps_complete from module.notification import PostNotification -from module.rss import analyser +from module.rss import RSSAnalyser, RSSEngine from .status import ProgramStatus @@ -17,21 +17,19 @@ class RSSThread(ProgramStatus): self._rss_thread = threading.Thread( target=self.rss_loop, ) + self.analyser = RSSAnalyser() def rss_loop(self): with DownloadClient() as client: client.init_downloader() while not self.stop_event.is_set(): - # Analyse RSS - with BangumiDatabase() as db: - new_data = analyser.rss_to_data(rss_link=settings.rss_link, database=db) - if new_data: - db.insert_list(new_data) - bangumi_list = db.not_added() - if bangumi_list: - with DownloadClient() as client: - client.set_rules(bangumi_list) - db.update_list(bangumi_list) + with DownloadClient() as client, RSSEngine() as engine: + # Analyse RSS + rss_list = engine.rss.search_combine() + for rss in rss_list: + self.analyser.rss_to_data(rss_link=rss.url, engine=engine) + # Run RSS Engine + engine.run(client) if settings.bangumi_manage.eps_complete: eps_complete() self.stop_event.wait(settings.program.rss_time) diff --git a/backend/src/module/database/combine.py b/backend/src/module/database/combine.py index e6dce3a7..25c717dd 100644 --- a/backend/src/module/database/combine.py +++ b/backend/src/module/database/combine.py @@ -3,10 +3,11 @@ from sqlmodel import Session, SQLModel from .rss import RSSDatabase from .torrent import TorrentDatabase from .bangumi import BangumiDatabase +from .engine import engine as e class Database(Session): - def __init__(self, engine): + def __init__(self, engine=e): self.engine = engine super().__init__(engine) self.rss = RSSDatabase(self) diff --git a/backend/src/module/database/rss.py b/backend/src/module/database/rss.py index bc539280..3f182877 100644 --- a/backend/src/module/database/rss.py +++ b/backend/src/module/database/rss.py @@ -36,6 +36,9 @@ class RSSDatabase: def search_active(self) -> list[RSSItem]: return self.session.exec(select(RSSItem).where(RSSItem.enabled)).all() + def search_combine(self) -> list[RSSItem]: + return self.session.exec(select(RSSItem).where(RSSItem.combine)).all() + def delete(self, _id: int): condition = delete(RSSItem).where(RSSItem.id == _id) self.session.exec(condition) diff --git a/backend/src/module/downloader/client/qb_downloader.py b/backend/src/module/downloader/client/qb_downloader.py index ea587daa..aae8d93f 100644 --- a/backend/src/module/downloader/client/qb_downloader.py +++ b/backend/src/module/downloader/client/qb_downloader.py @@ -82,10 +82,9 @@ class QbDownloader: status_filter=status_filter, category=category, tag=tag ) - def torrents_add(self, urls, save_path, category, torrent_files=None): + def torrents_add(self, torrent_files, save_path, category): resp = self._client.torrents_add( is_paused=False, - urls=urls, torrent_files=torrent_files, save_path=save_path, category=category, diff --git a/backend/src/module/downloader/download_client.py b/backend/src/module/downloader/download_client.py index 105f97d3..7d0a3ac4 100644 --- a/backend/src/module/downloader/download_client.py +++ b/backend/src/module/downloader/download_client.py @@ -1,7 +1,8 @@ import logging from module.conf import settings -from module.models import Bangumi +from module.models import Bangumi, Torrent +from module.network import RequestContent, TorrentInfo from .path import TorrentPath @@ -113,17 +114,25 @@ class DownloadClient(TorrentPath): self.client.torrents_delete(hashes) logger.info("[Downloader] Remove torrents.") - def add_torrent(self, torrent: dict): + def add_torrent( + self, torrent: Torrent | TorrentInfo | list, bangumi: Bangumi + ) -> bool: + if not bangumi.save_path: + bangumi.save_path = self._gen_save_path(bangumi) + with RequestContent() as req: + if isinstance(torrent, list): + torrent_file = [req.get_content(t.url) for t in torrent] + else: + torrent_file = req.get_content(torrent.url) if self.client.torrents_add( - urls=torrent.get("urls"), - torrent_files=torrent.get("torrent_files"), - save_path=torrent.get("save_path"), + torrent_files=torrent_file, + save_path=bangumi.save_path, category="Bangumi", ): - logger.debug(f"[Downloader] Add torrent: {torrent.get('save_path')}") + logger.debug(f"[Downloader] Add torrent: {bangumi.official_title}") return True else: - logger.error(f"[Downloader] Add torrent failed: {torrent.get('save_path')}") + logger.error(f"[Downloader] Add torrent failed: {bangumi.official_title}") return False def move_torrent(self, hashes, location): diff --git a/backend/src/module/manager/collector.py b/backend/src/module/manager/collector.py index b0bb9e15..1cc8726d 100644 --- a/backend/src/module/manager/collector.py +++ b/backend/src/module/manager/collector.py @@ -1,56 +1,43 @@ import logging -from module.database import BangumiDatabase from module.downloader import DownloadClient from module.models import Bangumi from module.searcher import SearchTorrent +from module.rss import RSSEngine logger = logging.getLogger(__name__) class SeasonCollector(DownloadClient): - def add_season_torrents(self, data: Bangumi, torrents, torrent_files=None): - if torrent_files: - download_info = { - "torrent_files": torrent_files, - "save_path": self._gen_save_path(data), - } - return self.add_torrent(download_info) - else: - download_info = { - "urls": [torrent.torrent_link for torrent in torrents], - "save_path": self._gen_save_path(data), - } - return self.add_torrent(download_info) + def add_season_torrents(self, bangumi: Bangumi, torrents: list): + return self.add_torrent(bangumi=bangumi, torrent=torrents) - def collect_season(self, data: Bangumi, link: str = None, proxy: bool = False): - logger.info(f"Start collecting {data.official_title} Season {data.season}...") + def collect_season(self, bangumi: Bangumi, link: str = None): + logger.info( + f"Start collecting {bangumi.official_title} Season {bangumi.season}..." + ) with SearchTorrent() as st: if not link: - torrents = st.search_season(data) + torrents = st.search_season(bangumi) else: - torrents = st.get_torrents(link, _filter="|".join(data.filter)) + torrents = st.get_torrents(link, _filter="|".join(bangumi.filter)) torrent_files = None - if proxy: - torrent_files = [ - st.get_content(torrent.torrent_link) for torrent in torrents - ] - return self.add_season_torrents( - data=data, torrents=torrents, torrent_files=torrent_files - ) + return self.add_season_torrents(bangumi=bangumi, torrents=torrents) - def subscribe_season(self, data: Bangumi): - with BangumiDatabase() as db: + @staticmethod + def subscribe_season(data: Bangumi): + with RSSEngine() as engine: data.added = True data.eps_collect = True - self.set_rule(data) - db.insert(data) - self.add_rss_feed(data.rss_link[0], item_path=data.official_title) + engine.add_rss( + rss_link=data.rss_link, name=data.official_title, combine=False + ) + engine.bangumi.add(data) def eps_complete(): - with BangumiDatabase() as bd: - datas = bd.not_complete() + with RSSEngine() as engine: + datas = engine.bangumi.not_complete() if datas: logger.info("Start collecting full season...") for data in datas: @@ -58,4 +45,4 @@ def eps_complete(): with SeasonCollector() as sc: sc.collect_season(data) data.eps_collect = True - bd.update_list(datas) + engine.bangumi.update_all(datas) diff --git a/backend/src/module/manager/torrent.py b/backend/src/module/manager/torrent.py index 3385e11e..facd8273 100644 --- a/backend/src/module/manager/torrent.py +++ b/backend/src/module/manager/torrent.py @@ -2,14 +2,14 @@ import logging from fastapi.responses import JSONResponse -from module.database import BangumiDatabase +from module.database import Database from module.downloader import DownloadClient from module.models import Bangumi logger = logging.getLogger(__name__) -class TorrentManager(BangumiDatabase): +class TorrentManager(Database): @staticmethod def __match_torrents_list(data: Bangumi) -> list: with DownloadClient() as client: @@ -28,12 +28,12 @@ class TorrentManager(BangumiDatabase): return f"Can't find {data.official_title} torrents." def delete_rule(self, _id: int | str, file: bool = False): - data = self.search_id(int(_id)) + data = self.bangumi.search_id(int(_id)) if isinstance(data, Bangumi): with DownloadClient() as client: client.remove_rule(data.rule_name) client.remove_rss_feed(data.official_title) - self.delete_one(int(_id)) + self.bangumi.delete_one(int(_id)) if file: torrent_message = self.delete_torrents(data, client) return JSONResponse( @@ -53,12 +53,12 @@ class TorrentManager(BangumiDatabase): ) def disable_rule(self, _id: str | int, file: bool = False): - data = self.search_id(int(_id)) + data = self.bangumi.search_id(int(_id)) if isinstance(data, Bangumi): with DownloadClient() as client: client.remove_rule(data.rule_name) data.deleted = True - self.update_one(data) + self.bangumi.update(data) if file: torrent_message = self.delete_torrents(data, client) return JSONResponse( @@ -80,10 +80,10 @@ class TorrentManager(BangumiDatabase): ) def enable_rule(self, _id: str | int): - data = self.search_id(int(_id)) + data = self.bangumi.search(int(_id)) if isinstance(data, Bangumi): data.deleted = False - self.update_one(data) + self.bangumi.update(data) with DownloadClient() as client: client.set_rule(data) logger.info(f"[Manager] Enable rule for {data.official_title}") @@ -99,7 +99,7 @@ class TorrentManager(BangumiDatabase): ) def update_rule(self, data: Bangumi): - old_data = self.search_id(data.id) + old_data = self.bangumi.search_id(data.id) if not old_data: logger.error(f"[Manager] Can't find data with {data.id}") return JSONResponse( @@ -115,7 +115,7 @@ class TorrentManager(BangumiDatabase): # Set new download rule client.remove_rule(data.rule_name) client.set_rule(data) - self.update_one(data) + self.bangumi.update(data) return JSONResponse( status_code=200, content={ @@ -124,13 +124,13 @@ class TorrentManager(BangumiDatabase): ) def search_all_bangumi(self): - datas = self.search_all() + datas = self.bangumi.search_all() if not datas: return [] return [data for data in datas if not data.deleted] def search_one(self, _id: int | str): - data = self.search_id(int(_id)) + data = self.bangumi.search_id(int(_id)) if not data: logger.error(f"[Manager] Can't find data with {_id}") return {"status": "error", "msg": f"Can't find data with {_id}"} diff --git a/backend/src/module/models/__init__.py b/backend/src/module/models/__init__.py index 587d548f..046930c8 100644 --- a/backend/src/module/models/__init__.py +++ b/backend/src/module/models/__init__.py @@ -1,4 +1,4 @@ -from .bangumi import Bangumi, Episode, BangumiUpdate +from .bangumi import Bangumi, Episode, BangumiUpdate, Notification from .config import Config from .rss import RSSItem, RSSUpdate from .torrent import EpisodeFile, SubtitleFile, Torrent, TorrentUpdate diff --git a/backend/src/module/network/request_contents.py b/backend/src/module/network/request_contents.py index c2e865fa..a231141b 100644 --- a/backend/src/module/network/request_contents.py +++ b/backend/src/module/network/request_contents.py @@ -13,7 +13,7 @@ from .site import mikan_parser @dataclass class TorrentInfo: name: str - torrent_link: str + url: str homepage: str _poster_link: str | None = None _official_title: str | None = None @@ -52,9 +52,7 @@ class RequestContent(RequestURL): ): if re.search(_filter, _title) is None: torrents.append( - TorrentInfo( - name=_title, torrent_link=torrent_url, homepage=homepage - ) + TorrentInfo(name=_title, url=torrent_url, homepage=homepage) ) return torrents except ConnectionError: diff --git a/backend/src/module/notification/notification.py b/backend/src/module/notification/notification.py index 29ac9d95..b50f6346 100644 --- a/backend/src/module/notification/notification.py +++ b/backend/src/module/notification/notification.py @@ -1,7 +1,7 @@ import logging from module.conf import settings -from module.database import BangumiDatabase +from module.database import Database from module.models import Notification from .plugin import ( @@ -36,8 +36,8 @@ class PostNotification: @staticmethod def _get_poster(notify: Notification): - with BangumiDatabase() as db: - poster_path = db.match_poster(notify.official_title) + with Database() as db: + poster_path = db.bangumi.match_poster(notify.official_title) if poster_path: poster_link = "https://mikanani.me" + poster_path else: diff --git a/backend/src/module/parser/title_parser.py b/backend/src/module/parser/title_parser.py index 4d1e7f00..c2b33fe2 100644 --- a/backend/src/module/parser/title_parser.py +++ b/backend/src/module/parser/title_parser.py @@ -39,7 +39,7 @@ class TitleParser: return official_title, tmdb_season, year @staticmethod - def raw_parser(raw: str, rss_link: str) -> Bangumi | None: + def raw_parser(raw: str) -> Bangumi | None: language = settings.rss_parser.language try: episode = raw_parser(raw) @@ -72,7 +72,6 @@ class TitleParser: eps_collect=False if episode.episode > 1 else True, offset=0, filter=settings.rss_parser.filter, - rss_link=[rss_link], ) logger.debug(f"RAW:{raw} >> {title_raw}") return data diff --git a/backend/src/module/rss/__init__.py b/backend/src/module/rss/__init__.py index f61d269a..70406ee3 100644 --- a/backend/src/module/rss/__init__.py +++ b/backend/src/module/rss/__init__.py @@ -1,3 +1,2 @@ from .analyser import RSSAnalyser - -analyser = RSSAnalyser() +from .engine import RSSEngine diff --git a/backend/src/module/rss/analyser.py b/backend/src/module/rss/analyser.py index 9746547d..f3124956 100644 --- a/backend/src/module/rss/analyser.py +++ b/backend/src/module/rss/analyser.py @@ -1,24 +1,22 @@ import logging import re +from .engine import RSSEngine + from module.conf import settings from module.models import Bangumi -from module.database import Database from module.network import RequestContent, TorrentInfo from module.parser import TitleParser logger = logging.getLogger(__name__) -class RSSAnalyser: - def __init__(self): - self._title_analyser = TitleParser() - +class RSSAnalyser(TitleParser): def official_title_parser(self, data: Bangumi, mikan_title: str): if settings.rss_parser.parser_type == "mikan": data.official_title = mikan_title if mikan_title else data.official_title elif settings.rss_parser.parser_type == "tmdb": - tmdb_title, season, year = self._title_analyser.tmdb_parser( + tmdb_title, season, year = self.tmdb_parser( data.official_title, data.season, settings.rss_parser.language ) data.official_title = tmdb_title @@ -29,7 +27,7 @@ class RSSAnalyser: data.official_title = re.sub(r"[/:.\\]", " ", data.official_title) @staticmethod - def get_rss_torrents(rss_link: str, full_parse: bool = True) -> list: + def get_rss_torrents(rss_link: str, full_parse: bool = True) -> list[TorrentInfo]: with RequestContent() as req: if full_parse: rss_torrents = req.get_torrents(rss_link) @@ -42,7 +40,7 @@ class RSSAnalyser: ) -> list: new_data = [] for torrent in torrents: - data = self._title_analyser.raw_parser(raw=torrent.name, rss_link=rss_link) + data = self.raw_parser(raw=torrent.name) if data and data.title_raw not in [i.title_raw for i in new_data]: try: poster_link, mikan_title = ( @@ -52,6 +50,7 @@ class RSSAnalyser: except AttributeError: poster_link, mikan_title = None, None data.poster_link = poster_link + data.rss_link = rss_link self.official_title_parser(data, mikan_title) if not full_parse: return [data] @@ -59,10 +58,8 @@ class RSSAnalyser: logger.debug(f"[RSS] New title found: {data.official_title}") return new_data - def torrent_to_data( - self, torrent: TorrentInfo, rss_link: str | None = None - ) -> Bangumi: - data = self._title_analyser.raw_parser(raw=torrent.name, rss_link=rss_link) + def torrent_to_data(self, torrent: TorrentInfo) -> Bangumi: + data = self.raw_parser(raw=torrent.name) if data: try: poster_link, mikan_title = ( @@ -76,16 +73,18 @@ class RSSAnalyser: return data def rss_to_data( - self, rss_link: str, database: Database, full_parse: bool = True + self, rss_link: str, engine: RSSEngine, full_parse: bool = True ) -> list[Bangumi]: rss_torrents = self.get_rss_torrents(rss_link, full_parse) - torrents_to_add = database.bangumi.match_list(rss_torrents, rss_link) + torrents_to_add = engine.bangumi.match_list(rss_torrents, rss_link) if not torrents_to_add: logger.debug("[RSS] No new title has been found.") return [] # New List new_data = self.torrents_to_data(torrents_to_add, rss_link, full_parse) if new_data: + # Add to database + engine.bangumi.add_all(new_data) return new_data else: return [] @@ -93,6 +92,6 @@ class RSSAnalyser: def link_to_data(self, link: str) -> Bangumi: torrents = self.get_rss_torrents(link, False) for torrent in torrents: - data = self.torrent_to_data(torrent, link) + data = self.torrent_to_data(torrent) if data: return data diff --git a/backend/src/module/rss/engine.py b/backend/src/module/rss/engine.py index 7ca5524e..d90ec2d8 100644 --- a/backend/src/module/rss/engine.py +++ b/backend/src/module/rss/engine.py @@ -1,6 +1,8 @@ import re import logging +from typing import Optional + from module.models import Bangumi, RSSItem, Torrent from module.network import RequestContent from module.downloader import DownloadClient @@ -23,12 +25,15 @@ class RSSEngine(Database): torrents.append( Torrent( name=torrent_info.name, - url=torrent_info.torrent_link, + url=torrent_info.url, homepage=torrent_info.homepage, ) ) return torrents + def get_combine_rss(self) -> list[RSSItem]: + return self.rss.get_combine() + def add_rss(self, rss_link: str, name: str | None = None, combine: bool = True): if not name: with RequestContent() as req: @@ -41,19 +46,15 @@ class RSSEngine(Database): new_torrents = self.torrent.check_new(torrents) return new_torrents - def match_torrent(self, torrent: Torrent): + def match_torrent(self, torrent: Torrent) -> Optional[Bangumi]: matched: Bangumi = self.bangumi.match_torrent(torrent.name) if matched: _filter = matched.filter.replace(",", "|") - if re.search(_filter, torrent.name, re.IGNORECASE): + if not re.search(_filter, torrent.name, re.IGNORECASE): torrent.refer_id = matched.id torrent.save_path = matched.save_path - with RequestContent() as req: - torrent_file = req.get_content(torrent.url) - return { - "torrent_files": torrent_file, - "save_path": torrent.save_path, - } + return matched + return None def run(self, client: DownloadClient): # Get All RSS Items @@ -63,13 +64,9 @@ class RSSEngine(Database): new_torrents = self.pull_rss(rss_item) # Get all enabled bangumi data for torrent in new_torrents: - download_info = self.match_torrent(torrent) - client.add_torrent(download_info) - torrent.downloaded = True + matched_data = self.match_torrent(torrent) + if matched_data: + if client.add_torrent(torrent, matched_data): + torrent.downloaded = True # Add all torrents to database self.torrent.add_all(new_torrents) - - -if __name__ == "__main__": - with RSSEngine() as engine: - engine.run() diff --git a/backend/src/module/searcher/searcher.py b/backend/src/module/searcher/searcher.py index dead9466..d18c3d9a 100644 --- a/backend/src/module/searcher/searcher.py +++ b/backend/src/module/searcher/searcher.py @@ -1,4 +1,4 @@ -from module.models import Bangumi, TorrentBase +from module.models import Bangumi, Torrent from module.network import RequestContent from module.searcher.plugin import search_url @@ -15,7 +15,7 @@ SEARCH_KEY = [ class SearchTorrent(RequestContent): def search_torrents( self, keywords: list[str], site: str = "mikan" - ) -> list[TorrentBase]: + ) -> list[Torrent]: url = search_url(site, keywords) # TorrentInfo to TorrentBase torrents = self.get_torrents(url) @@ -24,11 +24,11 @@ class SearchTorrent(RequestContent): for torrent in torrents: yield { "name": torrent.name, - "torrent_link": torrent.torrent_link, + "torrent_link": torrent.url, "homepage": torrent.homepage, } - return [TorrentBase(**d) for d in to_dict()] + return [Torrent(**d) for d in to_dict()] def search_season(self, data: Bangumi): keywords = [getattr(data, key) for key in SEARCH_KEY if getattr(data, key)] diff --git a/backend/src/module/update/data_migration.py b/backend/src/module/update/data_migration.py index f9739f98..95ccc905 100644 --- a/backend/src/module/update/data_migration.py +++ b/backend/src/module/update/data_migration.py @@ -1,7 +1,7 @@ import os from module.conf import LEGACY_DATA_PATH -from module.database import BangumiDatabase +from module.database import Database from module.models import Bangumi from module.utils import json_config @@ -15,8 +15,8 @@ def data_migration(): new_data = [] for info in infos: new_data.append(Bangumi(**info, rss_link=[rss_link])) - with BangumiDatabase() as database: - database.update_table() - database.insert_list(new_data) + with Database() as db: + db.create_table() + db.bangumi.add_all(new_data) LEGACY_DATA_PATH.unlink(missing_ok=True) From 4d2c4b80a7b263bb73f6a127d6043990ab9e33dd Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Sun, 6 Aug 2023 17:01:46 +0800 Subject: [PATCH 9/9] fix: fix bugs. Remove unnecessary code. --- backend/src/module/api/program.py | 1 - backend/src/module/core/program.py | 2 -- backend/src/module/core/rss_feed.py | 7 ------- backend/src/module/manager/torrent.py | 12 ++++-------- 4 files changed, 4 insertions(+), 18 deletions(-) diff --git a/backend/src/module/api/program.py b/backend/src/module/api/program.py index d75bd4ea..83720a0e 100644 --- a/backend/src/module/api/program.py +++ b/backend/src/module/api/program.py @@ -21,7 +21,6 @@ async def startup(): @router.on_event("shutdown") async def shutdown(): program.stop() - sys.exit(0) @router.get("/restart") diff --git a/backend/src/module/core/program.py b/backend/src/module/core/program.py index 855e52ff..22a2915f 100644 --- a/backend/src/module/core/program.py +++ b/backend/src/module/core/program.py @@ -3,7 +3,6 @@ import logging from module.conf import VERSION, settings from module.update import data_migration -from .rss_feed import add_rss_feed from .sub_thread import RenameThread, RSSThread logger = logging.getLogger(__name__) @@ -52,7 +51,6 @@ class Program(RenameThread, RSSThread): if self.enable_renamer: self.rename_start() if self.enable_rss: - add_rss_feed() self.rss_start() logger.info("Program running.") return {"status": "Program started."} diff --git a/backend/src/module/core/rss_feed.py b/backend/src/module/core/rss_feed.py index 82006602..2dae9d8c 100644 --- a/backend/src/module/core/rss_feed.py +++ b/backend/src/module/core/rss_feed.py @@ -25,10 +25,3 @@ def add_rss_feed(): if add: client.add_rss_feed(settings.rss_link) logger.info(f"Add RSS Feed: {settings.rss_link}") - - -if __name__ == "__main__": - from module.conf import setup_logger - - setup_logger() - add_rss_feed() diff --git a/backend/src/module/manager/torrent.py b/backend/src/module/manager/torrent.py index facd8273..72022d70 100644 --- a/backend/src/module/manager/torrent.py +++ b/backend/src/module/manager/torrent.py @@ -31,8 +31,9 @@ class TorrentManager(Database): data = self.bangumi.search_id(int(_id)) if isinstance(data, Bangumi): with DownloadClient() as client: - client.remove_rule(data.rule_name) - client.remove_rss_feed(data.official_title) + # client.remove_rule(data.rule_name) + # client.remove_rss_feed(data.official_title) + self.rss.delete(data.official_title) self.bangumi.delete_one(int(_id)) if file: torrent_message = self.delete_torrents(data, client) @@ -56,7 +57,7 @@ class TorrentManager(Database): data = self.bangumi.search_id(int(_id)) if isinstance(data, Bangumi): with DownloadClient() as client: - client.remove_rule(data.rule_name) + # client.remove_rule(data.rule_name) data.deleted = True self.bangumi.update(data) if file: @@ -84,8 +85,6 @@ class TorrentManager(Database): if isinstance(data, Bangumi): data.deleted = False self.bangumi.update(data) - with DownloadClient() as client: - client.set_rule(data) logger.info(f"[Manager] Enable rule for {data.official_title}") return JSONResponse( status_code=200, @@ -112,9 +111,6 @@ class TorrentManager(Database): path = client._gen_save_path(data) if match_list: client.move_torrent(match_list, path) - # Set new download rule - client.remove_rule(data.rule_name) - client.set_rule(data) self.bangumi.update(data) return JSONResponse( status_code=200,