diff --git a/backend/src/module/network/request_contents.py b/backend/src/module/network/request_contents.py index e271947e..4ea6767a 100644 --- a/backend/src/module/network/request_contents.py +++ b/backend/src/module/network/request_contents.py @@ -26,7 +26,9 @@ class RequestContent(RequestURL): if _filter is None: _filter = "|".join(settings.rss_parser.filter) for _title, torrent_url, homepage in parsed_items: - if re.search(_filter, _title) is None: + # A blank filter means "exclude nothing" — re.search("", x) matches + # every string, which would otherwise exclude everything. + if not _filter or re.search(_filter, _title) is None: torrents.append( Torrent(name=_title, url=torrent_url, homepage=homepage) ) diff --git a/backend/src/module/parser/analyser/mikan_parser.py b/backend/src/module/parser/analyser/mikan_parser.py index 109daaaf..e8eb47f4 100644 --- a/backend/src/module/parser/analyser/mikan_parser.py +++ b/backend/src/module/parser/analyser/mikan_parser.py @@ -1,5 +1,6 @@ import logging import re +from collections import OrderedDict from bs4 import BeautifulSoup from urllib3.util import parse_url @@ -9,8 +10,17 @@ from module.utils import save_image logger = logging.getLogger(__name__) -# In-memory cache for Mikan homepage lookups -_mikan_cache: dict[str, tuple[str, str]] = {} +# In-memory cache for Mikan homepage lookups. Keyed by per-episode homepage +# URL, so it is bounded (LRU-ish, oldest-evicted) rather than unlimited. +_MIKAN_CACHE_MAX = 512 +_mikan_cache: "OrderedDict[str, tuple[str, str]]" = OrderedDict() + + +def _cache_result(homepage: str, result: tuple[str, str]) -> tuple[str, str]: + if len(_mikan_cache) >= _MIKAN_CACHE_MAX: + _mikan_cache.popitem(last=False) + _mikan_cache[homepage] = result + return result async def mikan_parser(homepage: str): @@ -19,7 +29,10 @@ async def mikan_parser(homepage: str): root_path = parse_url(homepage).host async with RequestContent() as req: content = await req.get_html(homepage) - soup = BeautifulSoup(content, "html.parser") + # get_html returns None on a failed fetch; feed BeautifulSoup an empty + # string instead of None so a network failure surfaces as a normal + # "element not found" AttributeError rather than an uncaught TypeError. + soup = BeautifulSoup(content or "", "html.parser") poster_div = soup.find("div", {"class": "bangumi-poster"}).get("style") official_title = soup.select_one( 'p.bangumi-title a[href^="/Home/Bangumi/"]' @@ -30,13 +43,10 @@ async def mikan_parser(homepage: str): poster_path = poster_path.split("?")[0] img = await req.get_content(f"https://{root_path}{poster_path}") suffix = poster_path.split(".")[-1] - poster_link = save_image(img, suffix) - result = (poster_link, official_title) - _mikan_cache[homepage] = result - return result - result = ("", "") - _mikan_cache[homepage] = result - return result + # img can be None if the poster download failed; don't crash on it. + poster_link = save_image(img, suffix) if img else "" + return _cache_result(homepage, (poster_link, official_title)) + return _cache_result(homepage, ("", official_title)) if __name__ == '__main__': diff --git a/backend/src/module/parser/analyser/openai.py b/backend/src/module/parser/analyser/openai.py index eb31b595..103f921a 100644 --- a/backend/src/module/parser/analyser/openai.py +++ b/backend/src/module/parser/analyser/openai.py @@ -14,9 +14,9 @@ class Episode(BaseModel): title_en: Optional[str] title_zh: Optional[str] title_jp: Optional[str] - season: str + season: int season_raw: str - episode: str + episode: int sub: str group: str resolution: str diff --git a/backend/src/module/parser/analyser/tmdb_parser.py b/backend/src/module/parser/analyser/tmdb_parser.py index f926b3b4..7645d1aa 100644 --- a/backend/src/module/parser/analyser/tmdb_parser.py +++ b/backend/src/module/parser/analyser/tmdb_parser.py @@ -205,7 +205,10 @@ def get_season(seasons: list) -> tuple[int, str]: async def tmdb_parser(title, language, test: bool = False) -> TMDBInfo | None: - cache_key = f"{title}:{language}" + # `test` must be part of the key: test mode returns the raw remote poster + # URL instead of a locally-saved one, so mixing the two would poison + # whichever caller queries second. + cache_key = f"{title}:{language}:{test}" if cache_key in _tmdb_cache: return _tmdb_cache[cache_key] @@ -214,13 +217,13 @@ async def tmdb_parser(title, language, test: bool = False) -> TMDBInfo | None: contents = await req.get_json(url) if not contents: return None - contents = contents.get("results") - if contents.__len__() == 0: + contents = (contents or {}).get("results") or [] + if not contents: url = search_url(title.replace(" ", "")) contents_resp = await req.get_json(url) if not contents_resp: return None - contents = contents_resp.get("results") + contents = (contents_resp or {}).get("results") or [] # 判断动画 if contents: matched_id = None @@ -230,7 +233,8 @@ async def tmdb_parser(title, language, test: bool = False) -> TMDBInfo | None: matched_id = cid break if matched_id is None: - _tmdb_cache[cache_key] = None + # Don't cache the negative result permanently — a temporary + # TMDB hiccup shouldn't poison this title for the process lifetime. return None url_info = info_url(matched_id, language) info_content = await req.get_json(url_info) @@ -294,7 +298,8 @@ async def tmdb_parser(title, language, test: bool = False) -> TMDBInfo | None: img = await req.get_content( f"https://image.tmdb.org/t/p/w780{poster_path}" ) - poster_link = save_image(img, "jpg") + # img is None if the poster download failed; don't crash on it. + poster_link = save_image(img, "jpg") if img else None else: poster_link = "https://image.tmdb.org/t/p/w780" + poster_path else: @@ -318,9 +323,8 @@ async def tmdb_parser(title, language, test: bool = False) -> TMDBInfo | None: _tmdb_cache[cache_key] = result return result else: - if len(_tmdb_cache) >= _TMDB_CACHE_MAX: - _tmdb_cache.popitem(last=False) - _tmdb_cache[cache_key] = None + # No results at all — don't cache the negative result permanently, + # see the matched_id is None case above. return None diff --git a/backend/src/module/parser/title_parser.py b/backend/src/module/parser/title_parser.py index e19fc86e..35059b05 100644 --- a/backend/src/module/parser/title_parser.py +++ b/backend/src/module/parser/title_parser.py @@ -13,6 +13,19 @@ from module.parser.analyser import ( logger = logging.getLogger(__name__) +# Lazy singleton: building an OpenAIParser (and its AsyncOpenAI client) is not +# free, so keep one around and only rebuild it when the relevant settings change. +_openai_parser: OpenAIParser | None = None +_openai_parser_kwargs: dict | None = None + + +def _get_openai_parser(kwargs: dict) -> OpenAIParser: + global _openai_parser, _openai_parser_kwargs + if _openai_parser is None or _openai_parser_kwargs != kwargs: + _openai_parser = OpenAIParser(**kwargs) + _openai_parser_kwargs = kwargs + return _openai_parser + class TitleParser: def __init__(self): @@ -63,7 +76,7 @@ class TitleParser: # use OpenAI ChatGPT to parse raw title and get structured data if settings.experimental_openai.enable: kwargs = settings.experimental_openai.dict(exclude={"enable"}) - gpt = OpenAIParser(**kwargs) + gpt = _get_openai_parser(kwargs) episode_dict = await gpt.parse(raw, asdict=True) episode = Episode(**episode_dict) else: diff --git a/backend/src/module/rss/analyser.py b/backend/src/module/rss/analyser.py index b0929888..ffb72d04 100644 --- a/backend/src/module/rss/analyser.py +++ b/backend/src/module/rss/analyser.py @@ -13,16 +13,31 @@ logger = logging.getLogger(__name__) class RSSAnalyser: async def official_title_parser( - self, bangumi: Bangumi, rss: RSSItem, torrent: Torrent + self, + bangumi: Bangumi, + rss: RSSItem, + torrent: Torrent, + fetch_poster: bool = True, ): - if rss.parser == "mikan": - try: - bangumi.poster_link, bangumi.official_title = ( - await TitleParser.mikan_parser(torrent.homepage) - ) - except AttributeError: + if not fetch_poster: + pass + elif rss.parser == "mikan": + if not torrent.homepage: logger.warning("[Parser] Mikan torrent has no homepage info.") - pass + else: + try: + poster_link, official_title = await TitleParser.mikan_parser( + torrent.homepage + ) + except AttributeError as e: + logger.warning( + f"[Parser] Failed to parse Mikan homepage " + f"{torrent.homepage}: {e}" + ) + else: + bangumi.poster_link = poster_link + if official_title: + bangumi.official_title = official_title elif rss.parser == "tmdb": tmdb_title, season, year, poster_link = await TitleParser.tmdb_parser( bangumi.official_title, bangumi.season, settings.rss_parser.language @@ -56,6 +71,7 @@ class RSSAnalyser: await self.official_title_parser( bangumi=bangumi, rss=rss, torrent=torrent ) + bangumi.rss_link = rss.url if not full_parse: return [bangumi] seen_titles.add(bangumi.title_raw) @@ -63,10 +79,14 @@ class RSSAnalyser: logger.info(f"[RSS] New bangumi founded: {bangumi.official_title}") return new_data - async def torrent_to_data(self, torrent: Torrent, rss: RSSItem) -> Bangumi: + async def torrent_to_data( + self, torrent: Torrent, rss: RSSItem, fetch_poster: bool = True + ) -> Bangumi: bangumi = await TitleParser.raw_parser(raw=torrent.name) if bangumi: - await self.official_title_parser(bangumi=bangumi, rss=rss, torrent=torrent) + await self.official_title_parser( + bangumi=bangumi, rss=rss, torrent=torrent, fetch_poster=fetch_poster + ) bangumi.rss_link = rss.url return bangumi diff --git a/backend/src/module/rss/engine.py b/backend/src/module/rss/engine.py index 5169dfdb..e98d78d2 100644 --- a/backend/src/module/rss/engine.py +++ b/backend/src/module/rss/engine.py @@ -139,11 +139,11 @@ class RSSEngine: ) -> Optional[Bangumi]: matched = match_bangumi_in_list(torrent.name, bangumi_list) if matched: + torrent.bangumi_id = matched.id if matched.filter == "": return matched pattern = self._get_filter_pattern(matched.filter) if not pattern.search(torrent.name): - torrent.bangumi_id = matched.id return matched return None @@ -197,7 +197,9 @@ class RSSEngine: if matched_data: if await client.add_torrent(torrent, matched_data): logger.debug("[Engine] Add torrent %s to client", torrent.name) - torrent.downloaded = True + torrent.downloaded = True + # else: leave downloaded=False so a transient add failure + # (e.g. downloader unreachable) gets retried next tick. # Add all torrents to database await self.db.torrent.add_all(new_torrents) await self.db.commit() diff --git a/backend/src/module/searcher/provider.py b/backend/src/module/searcher/provider.py index 2fffccbf..e5f270d1 100644 --- a/backend/src/module/searcher/provider.py +++ b/backend/src/module/searcher/provider.py @@ -1,14 +1,20 @@ import re -from module.conf import SEARCH_CONFIG +# SEARCH_CONFIG is re-exported for existing importers (module.searcher.__init__); +# search_url itself reads get_provider() below so provider updates saved via +# save_provider() take effect immediately instead of only after a restart. +from module.conf.search_provider import SEARCH_CONFIG, get_provider # noqa: F401 from module.models import RSSItem def search_url(site: str, keywords: list[str]) -> RSSItem: keyword = "+".join(keywords) search_str = re.sub(r"[\W_ ]", "+", keyword) - if site in SEARCH_CONFIG.keys(): - url = re.sub(r"%s", search_str, SEARCH_CONFIG[site]) + providers = get_provider() + if site in providers: + # str.replace, not re.sub: the template is a literal "%s" placeholder, + # and search_str may itself contain regex-special characters. + url = providers[site].replace("%s", search_str) parser = "mikan" if site == "mikan" else "tmdb" rss_item = RSSItem( url=url, diff --git a/backend/src/module/searcher/searcher.py b/backend/src/module/searcher/searcher.py index 338f5ce4..bc7bcd2b 100644 --- a/backend/src/module/searcher/searcher.py +++ b/backend/src/module/searcher/searcher.py @@ -60,7 +60,13 @@ class SearchTorrent: for torrent in torrents: if len(exist_list) >= limit: break - bangumi = await self.analyser.torrent_to_data(torrent=torrent, rss=rss_item) + # Skip the per-torrent Mikan homepage fetch / poster download here: + # interactive search can return many results and doing that fetch + # serially for each one makes the search feel unresponsive. Poster + # is filled in afterwards from the (title-keyed, cached) TMDB lookup. + bangumi = await self.analyser.torrent_to_data( + torrent=torrent, rss=rss_item, fetch_poster=False + ) if bangumi: special_link = self.special_url(bangumi, site).url if special_link not in exist_list: diff --git a/backend/src/test/test_openai.py b/backend/src/test/test_openai.py index 4d2ee08e..47e36211 100644 --- a/backend/src/test/test_openai.py +++ b/backend/src/test/test_openai.py @@ -1,7 +1,8 @@ import json -import pytest from unittest import mock +import pytest + from module.parser.analyser.openai import DEFAULT_PROMPT, OpenAIParser diff --git a/backend/src/test/test_searcher.py b/backend/src/test/test_searcher.py index f1cd92c4..b6094a7f 100644 --- a/backend/src/test/test_searcher.py +++ b/backend/src/test/test_searcher.py @@ -1,12 +1,12 @@ """Tests for search providers: URL construction, keyword handling.""" -import pytest from unittest.mock import patch +import pytest + from module.models import Bangumi, RSSItem from module.searcher.provider import search_url - # --------------------------------------------------------------------------- # search_url # --------------------------------------------------------------------------- @@ -80,7 +80,7 @@ class TestSearchUrl: class TestSpecialUrl: def test_uses_bangumi_fields(self): """special_url builds keywords from SEARCH_KEY fields of Bangumi.""" - from module.searcher.searcher import SearchTorrent, SEARCH_KEY + from module.searcher.searcher import SEARCH_KEY, SearchTorrent from test.factories import make_bangumi bangumi = make_bangumi( diff --git a/backend/src/test/test_title_parser.py b/backend/src/test/test_title_parser.py index 012ac25f..1fc792fa 100644 --- a/backend/src/test/test_title_parser.py +++ b/backend/src/test/test_title_parser.py @@ -1,4 +1,5 @@ import pytest + from module.conf import settings from module.parser.title_parser import TitleParser diff --git a/backend/src/test/test_tmdb.py b/backend/src/test/test_tmdb.py index b8363a0c..e81805e4 100644 --- a/backend/src/test/test_tmdb.py +++ b/backend/src/test/test_tmdb.py @@ -1,7 +1,83 @@ +"""Tests for the TMDB parser. + +The default (mocked) test never touches the network: it patches +``RequestContent.get_json`` with fixture data so the suite stays +deterministic and offline. A separate live test exercises the real TMDB API +and is skipped unless explicitly opted into. +""" + +import importlib +import os + +import pytest + from module.parser.analyser.tmdb_parser import tmdb_parser +# `module.parser.analyser.__init__` re-exports the `tmdb_parser` function under +# the same name as this submodule, shadowing the submodule on the package +# object — so `import module.parser.analyser.tmdb_parser as x` would resolve +# to the function, not the module. Go through importlib to get the module. +tmdb_parser_module = importlib.import_module("module.parser.analyser.tmdb_parser") + +_SHOW_INFO = { + "genres": [{"id": 16, "name": "Animation"}], + "name": "冰海战记", + "original_name": "ヴィンランド・サガ", + "first_air_date": "2019-07-08", + "status": "Ended", + "poster_path": "/poster.jpg", + "seasons": [ + { + "name": "第 1 季", + "air_date": "2019-07-08", + "poster_path": "/s1.jpg", + "season_number": 1, + "episode_count": 24, + }, + { + "name": "第 2 季", + "air_date": "2023-01-09", + "poster_path": "/s2.jpg", + "season_number": 2, + "episode_count": 24, + }, + ], +} + + +async def _fake_get_json(url: str) -> dict: + if "/search/tv" in url: + return {"results": [{"id": 82684}]} + if "/season/" in url: + return {"episodes": []} + return _SHOW_INFO + + +async def test_tmdb_parser(mocker): + mocker.patch.object( + tmdb_parser_module.RequestContent, "get_json", side_effect=_fake_get_json + ) + tmdb_parser_module._tmdb_cache.clear() + + bangumi_title = "海盗战记" + bangumi_year = "2019" + bangumi_season = 2 + + tmdb_info = await tmdb_parser(bangumi_title, "zh", test=True) + + assert tmdb_info.title == "冰海战记" + assert tmdb_info.year == bangumi_year + assert tmdb_info.last_season == bangumi_season + + +@pytest.mark.live +@pytest.mark.skipif( + not os.environ.get("RUN_LIVE_TMDB_TESTS"), + reason="hits the real TMDB API; set RUN_LIVE_TMDB_TESTS=1 to run", +) +async def test_tmdb_parser_live(): + tmdb_parser_module._tmdb_cache.clear() -async def test_tmdb_parser(): bangumi_title = "海盗战记" bangumi_year = "2019" bangumi_season = 2 diff --git a/backend/src/test/test_torrent_parser.py b/backend/src/test/test_torrent_parser.py index 5f621146..036a1669 100644 --- a/backend/src/test/test_torrent_parser.py +++ b/backend/src/test/test_torrent_parser.py @@ -1,6 +1,7 @@ import sys import pytest + from module.parser.analyser import torrent_parser from module.parser.analyser.torrent_parser import get_path_basename