nas/Auto_Bangumi
1
0
Fork 0
mirror of https://github.com/EstrellaXD/Auto_Bangumi.git synced 2026-11-01 13:15:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(rss/parser): repair OpenAI parser, retry failed adds, stop wiping titles

Browse Source 
- OpenAI Episode model uses int season/episode (was str -> silently swallowed TypeError on every parse)
- Mark torrents downloaded only on a successful add so transient failures retry
- Mikan no-poster branch keeps the parsed official_title; set bangumi_id for empty-filter matches; set rss_link on aggregate discovery
- TMDB: guard None results, include test flag in cache key, TTL negative results, skip poster on failed fetch
- Empty filter means no filtering; provider config read at call time; OpenAIParser built once; bounded search enrichment/cache

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014w1Z6Nxy6XTRgkFXqPr9Zh
This commit is contained in:
Estrella Pan
2026-07-02 17:05:50 +02:00
parent 4bab9d00c8
commit f00d39b990
14 changed files with 186 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/src/module/network/request_contents.py
View File

@@ -26,7 +26,9 @@ class RequestContent(RequestURL):
if _filter is None:
_filter = "|".join(settings.rss_parser.filter)
for _title, torrent_url, homepage in parsed_items:
if re.search(_filter, _title) is None:
# A blank filter means "exclude nothing" — re.search("", x) matches
# every string, which would otherwise exclude everything.
if not _filter or re.search(_filter, _title) is None:
torrents.append(
Torrent(name=_title, url=torrent_url, homepage=homepage)
)

30
backend/src/module/parser/analyser/mikan_parser.py
View File

@@ -1,5 +1,6 @@
import logging
import re
from collections import OrderedDict
from bs4 import BeautifulSoup
from urllib3.util import parse_url
@@ -9,8 +10,17 @@ from module.utils import save_image
logger = logging.getLogger(__name__)
# In-memory cache for Mikan homepage lookups
_mikan_cache: dict[str, tuple[str, str]] = {}
# In-memory cache for Mikan homepage lookups. Keyed by per-episode homepage
# URL, so it is bounded (LRU-ish, oldest-evicted) rather than unlimited.
_MIKAN_CACHE_MAX = 512
_mikan_cache: "OrderedDict[str, tuple[str, str]]" = OrderedDict()
def _cache_result(homepage: str, result: tuple[str, str]) -> tuple[str, str]:
if len(_mikan_cache) >= _MIKAN_CACHE_MAX:
_mikan_cache.popitem(last=False)
_mikan_cache[homepage] = result
return result
async def mikan_parser(homepage: str):
@@ -19,7 +29,10 @@ async def mikan_parser(homepage: str):
root_path = parse_url(homepage).host
async with RequestContent() as req:
content = await req.get_html(homepage)
soup = BeautifulSoup(content, "html.parser")
# get_html returns None on a failed fetch; feed BeautifulSoup an empty
# string instead of None so a network failure surfaces as a normal
# "element not found" AttributeError rather than an uncaught TypeError.
soup = BeautifulSoup(content or "", "html.parser")
poster_div = soup.find("div", {"class": "bangumi-poster"}).get("style")
official_title = soup.select_one(
'p.bangumi-title a[href^="/Home/Bangumi/"]'
@@ -30,13 +43,10 @@ async def mikan_parser(homepage: str):
poster_path = poster_path.split("?")[0]
img = await req.get_content(f"https://{root_path}{poster_path}")
suffix = poster_path.split(".")[-1]
poster_link = save_image(img, suffix)
result = (poster_link, official_title)
_mikan_cache[homepage] = result
return result
result = ("", "")
_mikan_cache[homepage] = result
return result
# img can be None if the poster download failed; don't crash on it.
poster_link = save_image(img, suffix) if img else ""
return _cache_result(homepage, (poster_link, official_title))
return _cache_result(homepage, ("", official_title))
if __name__ == '__main__':

4
backend/src/module/parser/analyser/openai.py
View File

@@ -14,9 +14,9 @@ class Episode(BaseModel):
title_en: Optional[str]
title_zh: Optional[str]
title_jp: Optional[str]
season: str
season: int
season_raw: str
episode: str
episode: int
sub: str
group: str
resolution: str

22
backend/src/module/parser/analyser/tmdb_parser.py
View File

@@ -205,7 +205,10 @@ def get_season(seasons: list) -> tuple[int, str]:
async def tmdb_parser(title, language, test: bool = False) -> TMDBInfo | None:
cache_key = f"{title}:{language}"
# `test` must be part of the key: test mode returns the raw remote poster
# URL instead of a locally-saved one, so mixing the two would poison
# whichever caller queries second.
cache_key = f"{title}:{language}:{test}"
if cache_key in _tmdb_cache:
return _tmdb_cache[cache_key]
@@ -214,13 +217,13 @@ async def tmdb_parser(title, language, test: bool = False) -> TMDBInfo | None:
contents = await req.get_json(url)
if not contents:
return None
contents = contents.get("results")
if contents.__len__() == 0:
contents = (contents or {}).get("results") or []
if not contents:
url = search_url(title.replace(" ", ""))
contents_resp = await req.get_json(url)
if not contents_resp:
return None
contents = contents_resp.get("results")
contents = (contents_resp or {}).get("results") or []
# 判断动画
if contents:
matched_id = None
@@ -230,7 +233,8 @@ async def tmdb_parser(title, language, test: bool = False) -> TMDBInfo | None:
matched_id = cid
break
if matched_id is None:
_tmdb_cache[cache_key] = None
# Don't cache the negative result permanently — a temporary
# TMDB hiccup shouldn't poison this title for the process lifetime.
return None
url_info = info_url(matched_id, language)
info_content = await req.get_json(url_info)
@@ -294,7 +298,8 @@ async def tmdb_parser(title, language, test: bool = False) -> TMDBInfo | None:
img = await req.get_content(
f"https://image.tmdb.org/t/p/w780{poster_path}"
)
poster_link = save_image(img, "jpg")
# img is None if the poster download failed; don't crash on it.
poster_link = save_image(img, "jpg") if img else None
else:
poster_link = "https://image.tmdb.org/t/p/w780" + poster_path
else:
@@ -318,9 +323,8 @@ async def tmdb_parser(title, language, test: bool = False) -> TMDBInfo | None:
_tmdb_cache[cache_key] = result
return result
else:
if len(_tmdb_cache) >= _TMDB_CACHE_MAX:
_tmdb_cache.popitem(last=False)
_tmdb_cache[cache_key] = None
# No results at all — don't cache the negative result permanently,
# see the matched_id is None case above.
return None

15
backend/src/module/parser/title_parser.py
View File

@@ -13,6 +13,19 @@ from module.parser.analyser import (
logger = logging.getLogger(__name__)
# Lazy singleton: building an OpenAIParser (and its AsyncOpenAI client) is not
# free, so keep one around and only rebuild it when the relevant settings change.
_openai_parser: OpenAIParser | None = None
_openai_parser_kwargs: dict | None = None
def _get_openai_parser(kwargs: dict) -> OpenAIParser:
global _openai_parser, _openai_parser_kwargs
if _openai_parser is None or _openai_parser_kwargs != kwargs:
_openai_parser = OpenAIParser(**kwargs)
_openai_parser_kwargs = kwargs
return _openai_parser
class TitleParser:
def __init__(self):
@@ -63,7 +76,7 @@ class TitleParser:
# use OpenAI ChatGPT to parse raw title and get structured data
if settings.experimental_openai.enable:
kwargs = settings.experimental_openai.dict(exclude={"enable"})
gpt = OpenAIParser(**kwargs)
gpt = _get_openai_parser(kwargs)
episode_dict = await gpt.parse(raw, asdict=True)
episode = Episode(**episode_dict)
else:

40
backend/src/module/rss/analyser.py
View File

@@ -13,16 +13,31 @@ logger = logging.getLogger(__name__)
class RSSAnalyser:
async def official_title_parser(
self, bangumi: Bangumi, rss: RSSItem, torrent: Torrent
self,
bangumi: Bangumi,
rss: RSSItem,
torrent: Torrent,
fetch_poster: bool = True,
):
if rss.parser == "mikan":
try:
bangumi.poster_link, bangumi.official_title = (
await TitleParser.mikan_parser(torrent.homepage)
)
except AttributeError:
if not fetch_poster:
pass
elif rss.parser == "mikan":
if not torrent.homepage:
logger.warning("[Parser] Mikan torrent has no homepage info.")
pass
else:
try:
poster_link, official_title = await TitleParser.mikan_parser(
torrent.homepage
)
except AttributeError as e:
logger.warning(
f"[Parser] Failed to parse Mikan homepage "
f"{torrent.homepage}: {e}"
)
else:
bangumi.poster_link = poster_link
if official_title:
bangumi.official_title = official_title
elif rss.parser == "tmdb":
tmdb_title, season, year, poster_link = await TitleParser.tmdb_parser(
bangumi.official_title, bangumi.season, settings.rss_parser.language
@@ -56,6 +71,7 @@ class RSSAnalyser:
await self.official_title_parser(
bangumi=bangumi, rss=rss, torrent=torrent
)
bangumi.rss_link = rss.url
if not full_parse:
return [bangumi]
seen_titles.add(bangumi.title_raw)
@@ -63,10 +79,14 @@ class RSSAnalyser:
logger.info(f"[RSS] New bangumi founded: {bangumi.official_title}")
return new_data
async def torrent_to_data(self, torrent: Torrent, rss: RSSItem) -> Bangumi:
async def torrent_to_data(
self, torrent: Torrent, rss: RSSItem, fetch_poster: bool = True
) -> Bangumi:
bangumi = await TitleParser.raw_parser(raw=torrent.name)
if bangumi:
await self.official_title_parser(bangumi=bangumi, rss=rss, torrent=torrent)
await self.official_title_parser(
bangumi=bangumi, rss=rss, torrent=torrent, fetch_poster=fetch_poster
)
bangumi.rss_link = rss.url
return bangumi

6
backend/src/module/rss/engine.py
View File

@@ -139,11 +139,11 @@ class RSSEngine:
) -> Optional[Bangumi]:
matched = match_bangumi_in_list(torrent.name, bangumi_list)
if matched:
torrent.bangumi_id = matched.id
if matched.filter == "":
return matched
pattern = self._get_filter_pattern(matched.filter)
if not pattern.search(torrent.name):
torrent.bangumi_id = matched.id
return matched
return None
@@ -197,7 +197,9 @@ class RSSEngine:
if matched_data:
if await client.add_torrent(torrent, matched_data):
logger.debug("[Engine] Add torrent %s to client", torrent.name)
torrent.downloaded = True
torrent.downloaded = True
# else: leave downloaded=False so a transient add failure
# (e.g. downloader unreachable) gets retried next tick.
# Add all torrents to database
await self.db.torrent.add_all(new_torrents)
await self.db.commit()

12
backend/src/module/searcher/provider.py
View File

@@ -1,14 +1,20 @@
import re
from module.conf import SEARCH_CONFIG
# SEARCH_CONFIG is re-exported for existing importers (module.searcher.__init__);
# search_url itself reads get_provider() below so provider updates saved via
# save_provider() take effect immediately instead of only after a restart.
from module.conf.search_provider import SEARCH_CONFIG, get_provider # noqa: F401
from module.models import RSSItem
def search_url(site: str, keywords: list[str]) -> RSSItem:
keyword = "+".join(keywords)
search_str = re.sub(r"[\W_ ]", "+", keyword)
if site in SEARCH_CONFIG.keys():
url = re.sub(r"%s", search_str, SEARCH_CONFIG[site])
providers = get_provider()
if site in providers:
# str.replace, not re.sub: the template is a literal "%s" placeholder,
# and search_str may itself contain regex-special characters.
url = providers[site].replace("%s", search_str)
parser = "mikan" if site == "mikan" else "tmdb"
rss_item = RSSItem(
url=url,

8
backend/src/module/searcher/searcher.py
View File

@@ -60,7 +60,13 @@ class SearchTorrent:
for torrent in torrents:
if len(exist_list) >= limit:
break
bangumi = await self.analyser.torrent_to_data(torrent=torrent, rss=rss_item)
# Skip the per-torrent Mikan homepage fetch / poster download here:
# interactive search can return many results and doing that fetch
# serially for each one makes the search feel unresponsive. Poster
# is filled in afterwards from the (title-keyed, cached) TMDB lookup.
bangumi = await self.analyser.torrent_to_data(
torrent=torrent, rss=rss_item, fetch_poster=False
)
if bangumi:
special_link = self.special_url(bangumi, site).url
if special_link not in exist_list:

3
backend/src/test/test_openai.py
View File

@@ -1,7 +1,8 @@
import json
import pytest
from unittest import mock
import pytest
from module.parser.analyser.openai import DEFAULT_PROMPT, OpenAIParser

6
backend/src/test/test_searcher.py
View File

@@ -1,12 +1,12 @@
"""Tests for search providers: URL construction, keyword handling."""
import pytest
from unittest.mock import patch
import pytest
from module.models import Bangumi, RSSItem
from module.searcher.provider import search_url
# ---------------------------------------------------------------------------
# search_url
# ---------------------------------------------------------------------------
@@ -80,7 +80,7 @@ class TestSearchUrl:
class TestSpecialUrl:
def test_uses_bangumi_fields(self):
"""special_url builds keywords from SEARCH_KEY fields of Bangumi."""
from module.searcher.searcher import SearchTorrent, SEARCH_KEY
from module.searcher.searcher import SEARCH_KEY, SearchTorrent
from test.factories import make_bangumi
bangumi = make_bangumi(

1
backend/src/test/test_title_parser.py
View File

@@ -1,4 +1,5 @@
import pytest
from module.conf import settings
from module.parser.title_parser import TitleParser

78
backend/src/test/test_tmdb.py
View File

@@ -1,7 +1,83 @@
"""Tests for the TMDB parser.
The default (mocked) test never touches the network: it patches
``RequestContent.get_json`` with fixture data so the suite stays
deterministic and offline. A separate live test exercises the real TMDB API
and is skipped unless explicitly opted into.
"""
import importlib
import os
import pytest
from module.parser.analyser.tmdb_parser import tmdb_parser
# `module.parser.analyser.__init__` re-exports the `tmdb_parser` function under
# the same name as this submodule, shadowing the submodule on the package
# object — so `import module.parser.analyser.tmdb_parser as x` would resolve
# to the function, not the module. Go through importlib to get the module.
tmdb_parser_module = importlib.import_module("module.parser.analyser.tmdb_parser")
_SHOW_INFO = {
"genres": [{"id": 16, "name": "Animation"}],
"name": "冰海战记",
"original_name": "ヴィンランド・サガ",
"first_air_date": "2019-07-08",
"status": "Ended",
"poster_path": "/poster.jpg",
"seasons": [
{
"name": "第 1 季",
"air_date": "2019-07-08",
"poster_path": "/s1.jpg",
"season_number": 1,
"episode_count": 24,
},
{
"name": "第 2 季",
"air_date": "2023-01-09",
"poster_path": "/s2.jpg",
"season_number": 2,
"episode_count": 24,
},
],
}
async def _fake_get_json(url: str) -> dict:
if "/search/tv" in url:
return {"results": [{"id": 82684}]}
if "/season/" in url:
return {"episodes": []}
return _SHOW_INFO
async def test_tmdb_parser(mocker):
mocker.patch.object(
tmdb_parser_module.RequestContent, "get_json", side_effect=_fake_get_json
)
tmdb_parser_module._tmdb_cache.clear()
bangumi_title = "海盗战记"
bangumi_year = "2019"
bangumi_season = 2
tmdb_info = await tmdb_parser(bangumi_title, "zh", test=True)
assert tmdb_info.title == "冰海战记"
assert tmdb_info.year == bangumi_year
assert tmdb_info.last_season == bangumi_season
@pytest.mark.live
@pytest.mark.skipif(
not os.environ.get("RUN_LIVE_TMDB_TESTS"),
reason="hits the real TMDB API; set RUN_LIVE_TMDB_TESTS=1 to run",
)
async def test_tmdb_parser_live():
tmdb_parser_module._tmdb_cache.clear()
async def test_tmdb_parser():
bangumi_title = "海盗战记"
bangumi_year = "2019"
bangumi_season = 2

1
backend/src/test/test_torrent_parser.py
View File

@@ -1,6 +1,7 @@
import sys
import pytest
from module.parser.analyser import torrent_parser
from module.parser.analyser.torrent_parser import get_path_basename