From 4254651b631069ed6650a9208c56ce0738db8f5f Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Wed, 21 Jun 2023 15:48:24 +0800 Subject: [PATCH] temp: save --- .gitignore | 2 +- backend/src/module/database/bangumi.py | 16 ++++++++++- backend/src/module/database/orm/update.py | 34 ++++++++++++----------- backend/src/module/database/torrent.py | 15 +++++++--- backend/src/module/models/rss.py | 9 +++--- backend/src/module/rss/engine.py | 27 ++++++++---------- 6 files changed, 61 insertions(+), 42 deletions(-) diff --git a/.gitignore b/.gitignore index 1e5de80b..828f0945 100644 --- a/.gitignore +++ b/.gitignore @@ -178,6 +178,7 @@ test.* /backend/src/config/ /src/debuger.py /backend/src/dist.zip +/src # webui logs @@ -205,4 +206,3 @@ dist-ssr *.njsproj *.sln *.sw? - diff --git a/backend/src/module/database/bangumi.py b/backend/src/module/database/bangumi.py index 97efe1b4..c5983e89 100644 --- a/backend/src/module/database/bangumi.py +++ b/backend/src/module/database/bangumi.py @@ -169,7 +169,21 @@ class BangumiDatabase(Connector): dict_data = self.select.many(conditions=conditions, combine_operator="OR") return [self.__db_to_data(x) for x in dict_data] + def get_rss(self, rss_link: str) -> list[BangumiData]: + conditions = {"rss_link": rss_link} + dict_data = self.select.many(conditions=conditions, combine_operator="INSTR") + return [self.__db_to_data(x) for x in dict_data] + + def match_torrent(self, torrent_name: str, rss_link: str) -> BangumiData | None: + conditions = {"title_raw": torrent_name, "rss_link": rss_link} + dict_data = self.select.one(conditions=conditions, combine_operator="INSTR") + if not dict_data: + return None + return self.__db_to_data(dict_data) + if __name__ == "__main__": with BangumiDatabase() as db: - print(db.match_poster("久保")) + db.match_torrent( + "魔法科高校の劣等生 来訪者編", "https://bangumi.moe/rss/5f6b3e3e4e8c4b0001b2e3a3" + ) diff --git a/backend/src/module/database/orm/update.py b/backend/src/module/database/orm/update.py index 7b022418..000d5fe9 100644 --- a/backend/src/module/database/orm/update.py +++ b/backend/src/module/database/orm/update.py @@ -19,12 +19,11 @@ class Update: return self._connector.fetch() is not None def table(self): - columns = ", ".join( - [ - f"{key} {self.__python_to_sqlite_type(value)}" - for key, value in self._example_data.items() - ] - ) + columns_list = [ + self.__python_to_sqlite_type(key, value) + for key, value in self._example_data.items() + ] + columns = ", ".join(columns_list) create_table_sql = f"CREATE TABLE IF NOT EXISTS {self._table_name} ({columns});" self._connector.execute(create_table_sql) logger.debug(f"Create table {self._table_name}.") @@ -32,10 +31,10 @@ class Update: existing_columns = [x[1] for x in self._connector.fetch()] for key, value in self._example_data.items(): if key not in existing_columns: - insert_column = self.__python_to_sqlite_type(value) + insert_column = self.__python_to_sqlite_type(key, value) if value is None: value = "NULL" - add_column_sql = f"ALTER TABLE {self._table_name} ADD COLUMN {key} {insert_column} DEFAULT {value};" + add_column_sql = f"ALTER TABLE {self._table_name} ADD COLUMN {insert_column} DEFAULT {value};" self._connector.execute(add_column_sql) logger.debug(f"Update table {self._table_name}.") @@ -81,18 +80,21 @@ class Update: return True @staticmethod - def __python_to_sqlite_type(value) -> str: - if isinstance(value, int): - return "INTEGER NOT NULL" + def __python_to_sqlite_type(key, value) -> str: + if key == "id": + column = "INTEGER PRIMARY KEY" + elif isinstance(value, int): + column = "INTEGER NOT NULL" elif isinstance(value, float): - return "REAL NOT NULL" + column = "REAL NOT NULL" elif isinstance(value, str): - return "TEXT NOT NULL" + column = "TEXT NOT NULL" elif isinstance(value, bool): - return "INTEGER NOT NULL" + column = "INTEGER NOT NULL" elif isinstance(value, list): - return "TEXT NOT NULL" + column = "TEXT NOT NULL" elif value is None: - return "TEXT" + column = "TEXT" else: raise ValueError(f"Unsupported data type: {type(value)}") + return f"{key} {column}" diff --git a/backend/src/module/database/torrent.py b/backend/src/module/database/torrent.py index 2821182f..dd16823a 100644 --- a/backend/src/module/database/torrent.py +++ b/backend/src/module/database/torrent.py @@ -1,6 +1,6 @@ import logging -from .orm import Connector +from module.database.orm import Connector from module.models import TorrentData from module.conf import DATA_PATH @@ -10,9 +10,7 @@ logger = logging.getLogger(__name__) class TorrentDatabase(Connector): def __init__(self, database: str = DATA_PATH): super().__init__( - table_name="torrent", - data=TorrentData().dict(), - database=DATA_PATH + table_name="torrent", data=TorrentData().dict(), database=database ) def update_table(self): @@ -42,3 +40,12 @@ class TorrentDatabase(Connector): def get_all(self) -> list[TorrentData]: dict_datas = self.select.all() return [self.__db_to_data(data) for data in dict_datas] + + def get_torrent_name(self) -> list[str]: + dict_data = self.select.all() + return [data["name"] for data in dict_data] + + +if __name__ == "__main__": + with TorrentDatabase() as db: + db.update_table() diff --git a/backend/src/module/models/rss.py b/backend/src/module/models/rss.py index 66f38292..f9fe9aed 100644 --- a/backend/src/module/models/rss.py +++ b/backend/src/module/models/rss.py @@ -11,8 +11,7 @@ class RSSItem(BaseModel): class TorrentData(BaseModel): id: int = Field(0, alias="id") - name: str = Field(..., alias="name") - url: str = Field(..., alias="url") - matched: bool = Field(..., alias="matched") - downloaded: bool = Field(..., alias="downloaded") - save_path: str = Field(..., alias="save_path") + rss_id: int = Field(0, alias="rss_id") + name: str = Field("", alias="name") + url: str = Field("https://example.com/torrent", alias="url") + save_path: str = Field("path/to/save", alias="save_path") diff --git a/backend/src/module/rss/engine.py b/backend/src/module/rss/engine.py index b0c668b9..5fb7ff97 100644 --- a/backend/src/module/rss/engine.py +++ b/backend/src/module/rss/engine.py @@ -14,7 +14,7 @@ class RSSEngine(RequestContent): @staticmethod def _get_bangumi_data(rss_link: str) -> list[BangumiData]: with BangumiDatabase() as db: - return db.get_rss_data(rss_link) + return db.get_rss(rss_link) def add_rss(self, rss_link: str, name: str, combine: bool): if not name: @@ -28,9 +28,9 @@ class RSSEngine(RequestContent): return torrents @staticmethod - def match_torrent(torrent: TorrentInfo) -> TorrentData | None: + def match_torrent(torrent: TorrentInfo, rss_link: str) -> TorrentData | None: with BangumiDatabase() as db: - bangumi_data = db.match_torrent(torrent.name) + bangumi_data = db.match_torrent(torrent.name, rss_link) if bangumi_data: _filter = "|".join(bangumi_data.filter) if re.search(_filter, torrent.name): @@ -44,16 +44,13 @@ class RSSEngine(RequestContent): @staticmethod def filter_torrent(torrents: list[TorrentInfo]) -> list[TorrentInfo]: + new_torrents = [] with TorrentDatabase() as db: - in_db_torrents = db.get_all() - in_db_torrents = [x.name for x in in_db_torrents] - i = 0 - while i < len(torrents): - torrent = torrents[i] - if torrent.name in in_db_torrents: - torrents.pop(i) - i += 1 - return torrents + in_db_torrents: list = db.get_torrent_name() + for torrent in torrents: + if torrent.name not in in_db_torrents: + new_torrents.append(torrent) + return new_torrents def run(self): # Get All RSS Items @@ -61,10 +58,10 @@ class RSSEngine(RequestContent): # From RSS Items, get all torrents for rss_item in rss_items: torrents = self.get_torrents(rss_item.url) - self.filter_torrent(torrents) + new_torrents = self.filter_torrent(torrents) # Get all enabled bangumi data matched_torrents = [] - for torrent in torrents: + for torrent in new_torrents: matched_torrent = self.match_torrent(torrent) if matched_torrent: matched_torrents.append(matched_torrent) @@ -74,6 +71,6 @@ class RSSEngine(RequestContent): return matched_torrents -if __name__ == '__main__': +if __name__ == "__main__": with RSSEngine() as engine: engine.run()