feat(parser): per-provider parser choice, OpenAI as raw_parser fallback, honor group_tag

Store {url, parser} per search provider (backward-compatible with
legacy url-only configs) and use it in search_url() instead of the
hardcoded mikan-vs-tmdb rule.

TitleParser.raw_parser() now always tries the regex parser first and
only falls back to OpenAI when regex parsing fails and
experimental_openai.enable is set.

Renamer.gen_path() prefixes filenames with "[Group] " when
bangumi_manage.group_tag is enabled and a group tag was parsed.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GTnuEgR78cBk32bLNKar3z
This commit is contained in:
Estrella Pan
2026-07-02 19:00:18 +02:00
parent ca58fdc50e
commit 72c4a040e4
10 changed files with 420 additions and 34 deletions

View File

@@ -15,11 +15,11 @@ async def search_torrents(site: str = "mikan", keywords: str = Query(None)):
"""
if not keywords:
return []
keywords = keywords.split(" ")
keyword_list = keywords.split(" ")
async def event_generator():
st = SearchTorrent()
async for item in st.analyse_keyword(keywords=keywords, site=site):
async for item in st.analyse_keyword(keywords=keyword_list, site=site):
yield item
return EventSourceResponse(content=event_generator())
@@ -38,8 +38,12 @@ async def search_provider():
dependencies=[Depends(get_current_user)],
)
async def get_search_provider_config():
"""Get all search providers with their URL templates."""
return get_provider()
"""Get all search providers with their URL templates.
Each provider is stored internally as {url, parser}; only the URL is
exposed here to keep this endpoint's contract unchanged for callers.
"""
return {site: config["url"] for site, config in get_provider().items()}
@router.put(
@@ -50,4 +54,4 @@ async def get_search_provider_config():
async def update_search_provider_config(providers: dict[str, str]):
"""Update search providers configuration."""
save_provider(providers)
return get_provider()
return {site: config["url"] for site, config in get_provider().items()}

View File

@@ -1,33 +1,71 @@
from pathlib import Path
from typing import TypedDict
from module.utils import json_config
DEFAULT_PROVIDER = {
"mikan": "https://mikanani.me/RSS/Search?searchstr=%s",
"nyaa": "https://nyaa.si/?page=rss&q=%s&c=0_0&f=0",
"dmhy": "http://dmhy.org/topics/rss/rss.xml?keyword=%s",
class ProviderConfig(TypedDict):
"""单个搜索源配置:URL 模板与解析该源结果所用的解析器。"""
url: str
parser: str
def _default_parser(site: str) -> str:
"""旧配置没有 parser 字段时的默认规则:mikan 站点用 mikan 解析器,其余用 tmdb。"""
return "mikan" if site == "mikan" else "tmdb"
DEFAULT_PROVIDER: dict[str, ProviderConfig] = {
"mikan": {"url": "https://mikanani.me/RSS/Search?searchstr=%s", "parser": "mikan"},
"nyaa": {"url": "https://nyaa.si/?page=rss&q=%s&c=0_0&f=0", "parser": "tmdb"},
"dmhy": {"url": "http://dmhy.org/topics/rss/rss.xml?keyword=%s", "parser": "tmdb"},
}
PROVIDER_PATH = Path("config/search_provider.json")
def load_provider():
def _normalize(raw: dict) -> dict[str, ProviderConfig]:
"""兼容旧版本仅存储 URL 字符串的配置格式:{"site": "url"} -> {"site": {"url", "parser"}}。"""
normalized: dict[str, ProviderConfig] = {}
for site, value in raw.items():
if isinstance(value, str):
normalized[site] = {"url": value, "parser": _default_parser(site)}
else:
normalized[site] = {
"url": value.get("url", ""),
"parser": value.get("parser") or _default_parser(site),
}
return normalized
def load_provider() -> dict[str, ProviderConfig]:
if PROVIDER_PATH.exists():
return json_config.load(PROVIDER_PATH)
return _normalize(json_config.load(PROVIDER_PATH))
else:
PROVIDER_PATH.parent.mkdir(parents=True, exist_ok=True)
json_config.save(PROVIDER_PATH, DEFAULT_PROVIDER)
return DEFAULT_PROVIDER
def save_provider(providers: dict[str, str]):
"""Save search providers to config file and update SEARCH_CONFIG."""
def save_provider(providers: dict) -> None:
"""Save search providers to config file and update SEARCH_CONFIG.
Accepts either the legacy {site: url} shape or the {site: {url, parser}}
shape. When only a URL is supplied for a site that already has a saved
parser choice, that parser choice is preserved instead of being reset to
the default.
"""
global SEARCH_CONFIG
json_config.save(PROVIDER_PATH, providers)
SEARCH_CONFIG = providers
normalized = _normalize(providers)
for site, config in normalized.items():
if isinstance(providers.get(site), str) and site in SEARCH_CONFIG:
config["parser"] = SEARCH_CONFIG[site]["parser"]
json_config.save(PROVIDER_PATH, normalized)
SEARCH_CONFIG = normalized
def get_provider():
def get_provider() -> dict[str, ProviderConfig]:
"""Get current search providers config."""
return SEARCH_CONFIG

View File

@@ -79,19 +79,34 @@ class Renamer:
f"[Renamer] Episode offset {episode_offset} would make episode {original_episode} non-positive, ignoring offset"
)
episode = f"0{adjusted_episode}" if adjusted_episode < 10 else adjusted_episode
# 与旧版 downloader/path.py 的 rule_name 行为保持一致:开启 group_tag 时,
# 在文件名前加上 "[字幕组] " 前缀
group_prefix = (
f"[{file_info.group}] "
if settings.bangumi_manage.group_tag and file_info.group
else ""
)
if method == "none" or method == "subtitle_none":
return file_info.media_path
elif method == "pn":
return f"{file_info.title} S{season}E{episode}{file_info.suffix}"
return (
f"{group_prefix}{file_info.title} S{season}E{episode}{file_info.suffix}"
)
elif method == "advance":
return f"{bangumi_name} S{season}E{episode}{file_info.suffix}"
return f"{group_prefix}{bangumi_name} S{season}E{episode}{file_info.suffix}"
elif method == "normal":
logger.warning("[Renamer] Normal rename method is deprecated.")
return file_info.media_path
elif method == "subtitle_pn":
return f"{file_info.title} S{season}E{episode}.{file_info.language}{file_info.suffix}"
assert isinstance(
file_info, SubtitleFile
), "subtitle_pn requires a SubtitleFile"
return f"{group_prefix}{file_info.title} S{season}E{episode}.{file_info.language}{file_info.suffix}"
elif method == "subtitle_advance":
return f"{bangumi_name} S{season}E{episode}.{file_info.language}{file_info.suffix}"
assert isinstance(
file_info, SubtitleFile
), "subtitle_advance requires a SubtitleFile"
return f"{group_prefix}{bangumi_name} S{season}E{episode}.{file_info.language}{file_info.suffix}"
else:
logger.error(f"[Renamer] Unknown rename method: {method}")
return file_info.media_path
@@ -244,7 +259,7 @@ class Renamer:
logger.warning(f"[Renamer] {subtitle_path} rename failed")
@staticmethod
def _parse_bangumi_id_from_tags(tags: str) -> int | None:
def _parse_bangumi_id_from_tags(tags: str | None) -> int | None:
"""Extract bangumi_id from torrent tags.
Tags are comma-separated, and we look for 'ab:ID' format.

View File

@@ -81,16 +81,18 @@ class TitleParser:
async def raw_parser(raw: str) -> Bangumi | None:
language = settings.rss_parser.language
try:
# use OpenAI ChatGPT to parse raw title and get structured data
if settings.experimental_openai.enable:
# 优先使用正则解析;仅当正则解析失败且启用了 OpenAI 时,才用 OpenAI 兜底解析
episode = raw_parser(raw)
if episode is None:
if not settings.experimental_openai.enable:
return None
kwargs = settings.experimental_openai.dict(exclude={"enable"})
gpt = _get_openai_parser(kwargs)
episode_dict = await gpt.parse(raw, asdict=True)
episode = Episode(**episode_dict)
else:
episode = raw_parser(raw)
if episode is None:
return None
# asdict=True normally guarantees a dict; the `str` fallback
# case (JSON parse failure) intentionally raises TypeError
# here, caught by the except clause below.
episode = Episode(**episode_dict) # type: ignore[arg-type]
titles = {
"zh": episode.title_zh,

View File

@@ -14,8 +14,8 @@ def search_url(site: str, keywords: list[str]) -> RSSItem:
if site in providers:
# str.replace, not re.sub: the template is a literal "%s" placeholder,
# and search_str may itself contain regex-special characters.
url = providers[site].replace("%s", search_str)
parser = "mikan" if site == "mikan" else "tmdb"
url = providers[site]["url"].replace("%s", search_str)
parser = providers[site]["parser"]
rss_item = RSSItem(
url=url,
aggregate=False,

View File

@@ -118,8 +118,14 @@ class TestSearchProviderConfig:
def test_get_provider_config(self, authed_client):
"""GET /search/provider/config returns provider configurations."""
mock_providers = {
"mikan": "https://mikanani.me/RSS/Search?searchstr={keyword}",
"dmhy": "https://share.dmhy.org/search?keyword={keyword}",
"mikan": {
"url": "https://mikanani.me/RSS/Search?searchstr={keyword}",
"parser": "mikan",
},
"dmhy": {
"url": "https://share.dmhy.org/search?keyword={keyword}",
"parser": "tmdb",
},
}
with patch("module.api.search.get_provider", return_value=mock_providers):
response = authed_client.get("/api/v1/search/provider/config")
@@ -128,6 +134,8 @@ class TestSearchProviderConfig:
data = response.json()
assert "mikan" in data
assert "dmhy" in data
# Only the URL is exposed via this endpoint, not the parser field.
assert data["mikan"] == "https://mikanani.me/RSS/Search?searchstr={keyword}"
# ---------------------------------------------------------------------------
@@ -142,8 +150,18 @@ class TestUpdateProviderConfig:
"mikan": "https://mikanani.me/RSS/Search?searchstr={keyword}",
"custom": "https://custom.site/search?q={keyword}",
}
saved_config = {
"mikan": {
"url": "https://mikanani.me/RSS/Search?searchstr={keyword}",
"parser": "mikan",
},
"custom": {
"url": "https://custom.site/search?q={keyword}",
"parser": "tmdb",
},
}
with patch("module.api.search.save_provider") as mock_save:
with patch("module.api.search.get_provider", return_value=new_config):
with patch("module.api.search.get_provider", return_value=saved_config):
response = authed_client.put(
"/api/v1/search/provider/config", json=new_config
)
@@ -153,6 +171,7 @@ class TestUpdateProviderConfig:
data = response.json()
assert "mikan" in data
assert "custom" in data
assert data["custom"] == "https://custom.site/search?q={keyword}"
def test_update_provider_config_empty(self, authed_client):
"""PUT /search/provider/config with empty config."""

View File

@@ -4,6 +4,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from module.conf import settings
from module.downloader import DownloadClient
from module.manager.renamer import Renamer
from module.models import EpisodeFile, Notification, SubtitleFile
@@ -114,6 +115,103 @@ class TestGenPath:
assert result.endswith(".mp4")
# ---------------------------------------------------------------------------
# gen_path with group_tag
# ---------------------------------------------------------------------------
class TestGenPathWithGroupTag:
"""gen_path honors bangumi_manage.group_tag as a "[Group] " prefix,
mirroring the legacy downloader/path.py rule_name behavior."""
def test_group_tag_enabled_prefixes_pn_method(self):
"""pn method gets a "[Group] " prefix when group_tag is enabled."""
ep = EpisodeFile(
media_path="old.mkv",
group="SubGroup",
title="My Anime",
season=1,
episode=5,
suffix=".mkv",
)
with patch.object(settings.bangumi_manage, "group_tag", True):
result = Renamer.gen_path(ep, "Bangumi Name", method="pn")
assert result == "[SubGroup] My Anime S01E05.mkv"
def test_group_tag_enabled_prefixes_advance_method(self):
"""advance method gets a "[Group] " prefix when group_tag is enabled."""
ep = EpisodeFile(
media_path="old.mkv",
group="SubGroup",
title="My Anime",
season=1,
episode=5,
suffix=".mkv",
)
with patch.object(settings.bangumi_manage, "group_tag", True):
result = Renamer.gen_path(ep, "Bangumi Name", method="advance")
assert result == "[SubGroup] Bangumi Name S01E05.mkv"
def test_group_tag_enabled_prefixes_subtitle_methods(self):
"""subtitle_pn/subtitle_advance also get the prefix when enabled."""
sub = SubtitleFile(
media_path="sub.ass",
group="SubGroup",
title="My Anime",
season=1,
episode=5,
language="zh",
suffix=".ass",
)
with patch.object(settings.bangumi_manage, "group_tag", True):
result = Renamer.gen_path(sub, "Bangumi Name", method="subtitle_pn")
assert result == "[SubGroup] My Anime S01E05.zh.ass"
def test_group_tag_disabled_no_prefix(self):
"""No prefix is added when group_tag is disabled, even with a group."""
ep = EpisodeFile(
media_path="old.mkv",
group="SubGroup",
title="My Anime",
season=1,
episode=5,
suffix=".mkv",
)
with patch.object(settings.bangumi_manage, "group_tag", False):
result = Renamer.gen_path(ep, "Bangumi Name", method="pn")
assert result == "My Anime S01E05.mkv"
def test_group_tag_enabled_but_no_group_no_prefix(self):
"""No "[None] " prefix leaks in when group_tag is enabled but the
parsed file has no group (e.g. no [Group] tag in the torrent name)."""
ep = EpisodeFile(
media_path="old.mkv",
group=None,
title="My Anime",
season=1,
episode=5,
suffix=".mkv",
)
with patch.object(settings.bangumi_manage, "group_tag", True):
result = Renamer.gen_path(ep, "Bangumi Name", method="pn")
assert result == "My Anime S01E05.mkv"
def test_group_tag_enabled_none_and_subtitle_none_unaffected(self):
"""none/subtitle_none methods return the original path unchanged,
regardless of group_tag."""
ep = EpisodeFile(
media_path="original/path/file.mkv",
group="SubGroup",
title="Test",
season=1,
episode=1,
suffix=".mkv",
)
with patch.object(settings.bangumi_manage, "group_tag", True):
result = Renamer.gen_path(ep, "Bangumi", method="none")
assert result == "original/path/file.mkv"
# ---------------------------------------------------------------------------
# rename_file
# ---------------------------------------------------------------------------

View File

@@ -0,0 +1,101 @@
"""Tests for module.conf.search_provider: {url, parser} storage and
backward-compatible loading of legacy url-only provider configs."""
import pytest
from module.conf import search_provider as search_provider_module
from module.conf.search_provider import _normalize, load_provider, save_provider
@pytest.fixture(autouse=True)
def _isolate_provider_state(tmp_path, monkeypatch):
"""Point PROVIDER_PATH at a scratch file and restore SEARCH_CONFIG after
each test, so tests never touch the real config/search_provider.json or
leak module-level state into other tests."""
monkeypatch.setattr(
search_provider_module, "PROVIDER_PATH", tmp_path / "search_provider.json"
)
original = search_provider_module.SEARCH_CONFIG
yield
search_provider_module.SEARCH_CONFIG = original
class TestNormalize:
def test_normalize_legacy_string_config_migrates_to_dict(self):
"""Old {"site": "url"} entries gain a default parser field."""
legacy = {
"mikan": "https://mikanani.me/RSS/Search?searchstr=%s",
"nyaa": "https://nyaa.si/?page=rss&q=%s",
}
result = _normalize(legacy)
assert result["mikan"] == {
"url": "https://mikanani.me/RSS/Search?searchstr=%s",
"parser": "mikan",
}
assert result["nyaa"] == {
"url": "https://nyaa.si/?page=rss&q=%s",
"parser": "tmdb",
}
def test_normalize_new_dict_config_preserves_parser(self):
"""Already-migrated {"site": {"url", "parser"}} entries pass through."""
new_format = {"custom": {"url": "https://x.example/%s", "parser": "tmdb"}}
result = _normalize(new_format)
assert result == new_format
def test_normalize_missing_parser_field_defaults(self):
"""A dict entry missing "parser" falls back to the default rule."""
result = _normalize({"custom": {"url": "https://x.example/%s"}})
assert result["custom"]["parser"] == "tmdb"
class TestLoadProvider:
def test_load_provider_backward_compatible_with_legacy_file(
self, tmp_path, monkeypatch
):
"""A pre-existing url-only JSON config file still loads successfully."""
from module.utils import json_config
legacy_path = tmp_path / "search_provider.json"
json_config.save(
legacy_path,
{"mikan": "https://mikanani.me/RSS/Search?searchstr=%s"},
)
monkeypatch.setattr(search_provider_module, "PROVIDER_PATH", legacy_path)
result = load_provider()
assert result["mikan"]["url"] == "https://mikanani.me/RSS/Search?searchstr=%s"
assert result["mikan"]["parser"] == "mikan"
def test_load_provider_creates_default_file_when_missing(self, tmp_path):
"""A missing config file falls back to DEFAULT_PROVIDER and persists it."""
result = load_provider()
assert result == search_provider_module.DEFAULT_PROVIDER
assert search_provider_module.PROVIDER_PATH.exists()
class TestSaveProvider:
def test_save_provider_stores_url_and_parser(self):
"""Saving a full {url, parser} config round-trips through SEARCH_CONFIG."""
save_provider({"custom": {"url": "https://x.example/%s", "parser": "tmdb"}})
assert search_provider_module.get_provider()["custom"] == {
"url": "https://x.example/%s",
"parser": "tmdb",
}
def test_save_provider_url_only_update_preserves_existing_parser(self):
"""Saving just a URL for a site with a saved custom parser keeps that
parser instead of resetting it to the site-name-based default."""
save_provider({"custom": {"url": "https://old.example/%s", "parser": "mikan"}})
save_provider({"custom": "https://new.example/%s"})
result = search_provider_module.get_provider()["custom"]
assert result["url"] == "https://new.example/%s"
assert result["parser"] == "mikan"
def test_save_provider_url_only_new_site_uses_default_parser(self):
"""A brand-new site saved as a bare URL gets the default parser rule."""
save_provider({"custom": "https://x.example/%s"})
assert search_provider_module.get_provider()["custom"]["parser"] == "tmdb"

View File

@@ -72,6 +72,38 @@ class TestSearchUrl:
assert result.aggregate is False
class TestSearchUrlPerProviderParser:
"""search_url reads the parser from each provider's own {url, parser}
config instead of the old hardcoded "mikan-or-tmdb" rule."""
def test_uses_providers_declared_parser(self):
"""A custom provider's declared parser is honored, not the default."""
with patch(
"module.searcher.provider.get_provider",
return_value={
"custom": {"url": "https://custom.example/?q=%s", "parser": "openai"}
},
):
result = search_url("custom", ["test"])
assert result.parser == "openai"
def test_mikan_site_uses_declared_parser_not_hardcoded(self):
"""Even the "mikan" site name honors the configured parser value."""
with patch(
"module.searcher.provider.get_provider",
return_value={
"mikan": {
"url": "https://mikanani.me/RSS/Search?searchstr=%s",
"parser": "tmdb",
}
},
):
result = search_url("mikan", ["test"])
assert result.parser == "tmdb"
# ---------------------------------------------------------------------------
# SearchTorrent.special_url
# ---------------------------------------------------------------------------

View File

@@ -1,3 +1,5 @@
from unittest.mock import AsyncMock, patch
import pytest
from module.conf import settings
@@ -9,6 +11,7 @@ class TestTitleParser:
async def test_parse_without_openai(self):
text = "[梦蓝字幕组]New Doraemon 哆啦A梦新番[747][2023.02.25][AVC][1080P][GB_JP][MP4]"
result = await TitleParser.raw_parser(text)
assert result is not None
assert result.group_name == "梦蓝字幕组"
assert result.title_raw == "New Doraemon"
assert result.dpi == "1080P"
@@ -22,6 +25,7 @@ class TestTitleParser:
async def test_parse_with_openai(self):
text = "[梦蓝字幕组]New Doraemon 哆啦A梦新番[747][2023.02.25][AVC][1080P][GB_JP][MP4]"
result = await TitleParser.raw_parser(text)
assert result is not None
assert result.group_name == "梦蓝字幕组"
assert result.title_raw == "New Doraemon"
assert result.dpi == "1080P"
@@ -29,6 +33,77 @@ class TestTitleParser:
assert result.subtitle == "GB_JP"
class TestRawParserOpenAIFallback:
"""OpenAI is only consulted as a rescue when the regex raw_parser fails,
never as a replacement for it (see title_parser.py:76-84)."""
def teardown_method(self):
title_parser_module.reset_cache()
def _make_episode_dict(self) -> dict:
return {
"title_en": "OpenAI Title",
"title_zh": None,
"title_jp": None,
"season": 1,
"season_raw": "S1",
"episode": 1,
"sub": "GB",
"group": "OpenAIGroup",
"resolution": "1080P",
"source": "Web",
}
async def test_raw_parser_regex_success_openai_enabled_skips_openai(self):
"""When the regex parser succeeds, OpenAI must not be invoked even
if it is enabled — the LLM is a fallback, not a replacement."""
text = "[梦蓝字幕组]New Doraemon 哆啦A梦新番[747][2023.02.25][AVC][1080P][GB_JP][MP4]"
with patch.object(settings.experimental_openai, "enable", True):
with patch(
"module.parser.title_parser._get_openai_parser"
) as mock_get_parser:
result = await TitleParser.raw_parser(text)
assert result is not None
assert result.title_raw == "New Doraemon"
mock_get_parser.assert_not_called()
async def test_raw_parser_regex_failure_openai_enabled_falls_back_to_openai(self):
"""When the regex parser returns None and OpenAI is enabled, OpenAI
is tried as a rescue and its result is used."""
mock_gpt = AsyncMock()
mock_gpt.parse = AsyncMock(return_value=self._make_episode_dict())
with patch.object(settings.experimental_openai, "enable", True):
with patch(
"module.parser.title_parser.raw_parser", return_value=None
) as mock_raw_parser:
with patch(
"module.parser.title_parser._get_openai_parser",
return_value=mock_gpt,
) as mock_get_parser:
result = await TitleParser.raw_parser("unparseable garbage title")
mock_raw_parser.assert_called_once()
mock_get_parser.assert_called_once()
mock_gpt.parse.assert_awaited_once()
assert result is not None
assert result.official_title == "OpenAI Title"
assert result.group_name == "OpenAIGroup"
async def test_raw_parser_regex_failure_openai_disabled_returns_none(self):
"""When the regex parser fails and OpenAI is disabled, no fallback
happens and the method returns None."""
with patch.object(settings.experimental_openai, "enable", False):
with patch("module.parser.title_parser.raw_parser", return_value=None):
with patch(
"module.parser.title_parser._get_openai_parser"
) as mock_get_parser:
result = await TitleParser.raw_parser("unparseable garbage title")
assert result is None
mock_get_parser.assert_not_called()
class TestOpenAIParserResetCache:
"""reset_cache() must drop the lazy OpenAI-parser singleton on config
reload, or a changed api_key/api_base keeps using the old client."""
@@ -37,7 +112,9 @@ class TestOpenAIParserResetCache:
title_parser_module.reset_cache()
def test_reset_cache_clears_singleton(self):
title_parser_module._openai_parser = object()
# A cheap non-None sentinel; the test only cares that reset_cache()
# clears it, not that it's a real OpenAIParser.
title_parser_module._openai_parser = object() # type: ignore[assignment]
title_parser_module._openai_parser_kwargs = {"api_key": "old"}
title_parser_module.reset_cache()