From d251cc574629dd63c1f3d7cd577fa95596deb9d0 Mon Sep 17 00:00:00 2001 From: Estrella Pan Date: Sat, 11 Jul 2026 17:59:01 +0200 Subject: [PATCH] fix(rss): preserve typed media identities --- backend/src/module/database/bangumi.py | 145 ++++++++++++------ backend/src/module/database/movie.py | 6 +- backend/src/module/rss/analyser.py | 13 +- backend/src/test/test_database.py | 72 ++++++++- .../test/test_rss_analyser_typed_identity.py | 46 ++++++ 5 files changed, 226 insertions(+), 56 deletions(-) create mode 100644 backend/src/test/test_rss_analyser_typed_identity.py diff --git a/backend/src/module/database/bangumi.py b/backend/src/module/database/bangumi.py index 5782e2cd..36f2b6df 100644 --- a/backend/src/module/database/bangumi.py +++ b/backend/src/module/database/bangumi.py @@ -1,7 +1,7 @@ import json import logging import re -from typing import Optional +from typing import TYPE_CHECKING, Optional from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.sql import func @@ -9,6 +9,9 @@ from sqlmodel import and_, delete, false, or_, select from module.models import Bangumi, BangumiUpdate +if TYPE_CHECKING: + from module.parser.analyser.tokenizer.result import ParsedRelease + logger = logging.getLogger(__name__) @@ -110,12 +113,16 @@ def _find_semantic_match(data: Bangumi, candidates: list[Bangumi]) -> Optional[B is_exact_duplicate = ( candidate.title_raw == data.title_raw and candidate.group_name == data.group_name + and candidate.season == data.season + and candidate.episode_type == data.episode_type ) if is_exact_duplicate: continue is_semantic_match = ( - candidate.dpi == data.dpi + candidate.season == data.season + and candidate.episode_type == data.episode_type + and candidate.dpi == data.dpi and candidate.subtitle == data.subtitle and candidate.source == data.source and _groups_are_similar(candidate.group_name, data.group_name) @@ -151,18 +158,51 @@ def match_bangumi_in_list( (the job the old module-level TTL cache used to do). Returns the bangumi with the longest matching pattern for specificity. """ + # Lazy import avoids the database -> parser -> database import cycle during + # application startup while still giving matching the typed parser result. + from module.parser.analyser.tokenizer import parse_release_title + + release = parse_release_title(torrent_name) best_match: Optional[Bangumi] = None - best_match_len = 0 + best_rank = (-1, -1, -1) for bangumi in bangumi_list: - if bangumi.deleted: + if bangumi.deleted or not _release_matches_bangumi(release, bangumi): continue for pattern in _all_title_patterns(bangumi): - if pattern in torrent_name and len(pattern) > best_match_len: + if pattern not in torrent_name: + continue + group_match = int( + release is not None + and _groups_are_similar(release.group, bangumi.group_name) + ) + rank = (len(pattern), group_match, bangumi.season) + if rank > best_rank: best_match = bangumi - best_match_len = len(pattern) + best_rank = rank return best_match +def _release_matches_bangumi(release: "ParsedRelease | None", bangumi: Bangumi) -> bool: + if release is None: + return True + + from module.parser.analyser.tokenizer import MediaType + + if release.media_type in {MediaType.PV, MediaType.OPENING, MediaType.ENDING}: + return False + if release.media_type is MediaType.MOVIE: + expected_type = "movie" + elif release.media_type in {MediaType.OVA, MediaType.OAD, MediaType.SPECIAL}: + expected_type = "special" + else: + expected_type = "episode" + if bangumi.episode_type != expected_type: + return False + if expected_type == "special": + return bangumi.season == 0 + return release.season is None or bangumi.season == release.season + + class BangumiDatabase: def __init__(self, session: AsyncSession): self.session = session @@ -247,7 +287,7 @@ class BangumiDatabase: return _all_title_patterns(bangumi) async def find_duplicate(self, data: Bangumi) -> Optional[Bangumi]: - """Find an existing bangumi with the same title_raw and group_name. + """Find an existing rule with the same typed subscription identity. Legacy data may contain multiple rows for the same key; return the oldest (lowest id) deterministically and warn instead of raising. @@ -258,6 +298,8 @@ class BangumiDatabase: and_( Bangumi.title_raw == data.title_raw, Bangumi.group_name == data.group_name, + Bangumi.season == data.season, + Bangumi.episode_type == data.episode_type, ) ) .order_by(Bangumi.id) # type: ignore[arg-type] @@ -266,10 +308,13 @@ class BangumiDatabase: rows = list(result.scalars().all()) if len(rows) > 1: logger.warning( - "Multiple bangumi rows share (title_raw=%s, group_name=%s); " + "Multiple bangumi rows share (title_raw=%s, group_name=%s, " + "season=%s, episode_type=%s); " "using the oldest (id=%s)", data.title_raw, data.group_name, + data.season, + data.episode_type, rows[0].id, ) return rows[0] if rows else None @@ -310,24 +355,38 @@ class BangumiDatabase: if not datas: return 0 - # Batch query: get all existing (title_raw, group_name) combinations in one query + # Batch query: load all existing typed subscription identities at once. # This replaces N individual _is_duplicate() calls with a single SELECT - keys_to_check = [(d.title_raw, d.group_name) for d in datas] + keys_to_check = [ + (d.title_raw, d.group_name, d.season, d.episode_type) for d in datas + ] conditions = [ - and_(Bangumi.title_raw == tr, Bangumi.group_name == gn) - for tr, gn in keys_to_check + and_( + Bangumi.title_raw == tr, + Bangumi.group_name == gn, + Bangumi.season == season, + Bangumi.episode_type == episode_type, + ) + for tr, gn, season, episode_type in keys_to_check ] if conditions: - statement = select(Bangumi.title_raw, Bangumi.group_name).where( - or_(*conditions) - ) + statement = select( + Bangumi.title_raw, + Bangumi.group_name, + Bangumi.season, + Bangumi.episode_type, + ).where(or_(*conditions)) result = await self.session.execute(statement) existing = set(result.all()) else: existing = set() # Filter out exact duplicates - to_add = [d for d in datas if (d.title_raw, d.group_name) not in existing] + to_add = [ + d + for d in datas + if (d.title_raw, d.group_name, d.season, d.episode_type) not in existing + ] # Batch query: load all semantic-duplicate candidates (grouped by # official_title) in one SELECT, instead of one find_semantic_duplicate() @@ -376,7 +435,7 @@ class BangumiDatabase: seen = set() unique_to_add = [] for d in really_to_add: - key = (d.title_raw, d.group_name) + key = (d.title_raw, d.group_name, d.season, d.episode_type) if key not in seen: seen.add(key) unique_to_add.append(d) @@ -506,9 +565,17 @@ class BangumiDatabase: return bangumi async def search_official_title(self, official_title: str) -> Optional[Bangumi]: - statement = select(Bangumi).where(Bangumi.official_title == official_title) + statement = ( + select(Bangumi) + .where(Bangumi.official_title == official_title) + .order_by( + Bangumi.episode_type, # type: ignore[arg-type] + Bangumi.season, # type: ignore[arg-type] + Bangumi.id, # type: ignore[arg-type] + ) + ) result = await self.session.execute(statement) - return result.scalar_one_or_none() + return result.scalars().first() async def search_ids(self, ids: list[int]) -> list[Bangumi]: """Batch lookup multiple bangumi by their IDs.""" @@ -531,41 +598,21 @@ class BangumiDatabase: if not match_datas: return torrent_list - # Build index for O(1) lookup after regex match - # Include both title_raw and all aliases - title_index: dict[str, Bangumi] = {} - for m in match_datas: - # Add main title_raw (skip if None to avoid TypeError in sorted()) - if m.title_raw: - title_index[m.title_raw] = m - # Add all aliases - for alias in _get_aliases_list(m): - if alias: - title_index[alias] = m - - # Build compiled regex pattern for fast substring matching - # Sort by length descending so longer (more specific) matches are found first - sorted_titles = sorted(title_index.keys(), key=len, reverse=True) - # Escape special regex characters and join with alternation - pattern = "|".join(re.escape(title) for title in sorted_titles) - title_regex = re.compile(pattern) - unmatched = [] - rss_updated = set() + rss_updated: set[int | tuple[str, int, str]] = set() for torrent in torrent_list: - match = title_regex.search(torrent.name) - if match: - matched_title = match.group(0) - match_data = title_index[matched_title] - # Use the bangumi's main title_raw for rss_updated tracking - if ( - rss_link not in match_data.rss_link - and match_data.title_raw not in rss_updated - ): + match_data = match_bangumi_in_list(torrent.name, match_datas) + if match_data is not None: + match_key: int | tuple[str, int, str] = match_data.id or ( + match_data.title_raw, + match_data.season, + match_data.episode_type, + ) + if rss_link not in match_data.rss_link and match_key not in rss_updated: match_data = await self.session.merge(match_data) match_data.rss_link += f",{rss_link}" match_data.added = False - rss_updated.add(match_data.title_raw) + rss_updated.add(match_key) else: unmatched.append(torrent) # Batch commit all rss_link updates diff --git a/backend/src/module/database/movie.py b/backend/src/module/database/movie.py index 6848bcbb..8fe2f995 100644 --- a/backend/src/module/database/movie.py +++ b/backend/src/module/database/movie.py @@ -124,6 +124,9 @@ class MovieDatabase: logger.debug("Enable movie %s.", movie.title_raw) async def match_list(self, torrent_list: list, rss_link: str) -> list: + # Lazy import avoids a database/parser import cycle during startup. + from module.parser.analyser.tokenizer import MediaType, parse_release_title + match_datas = await self.search_all() if not match_datas: return torrent_list @@ -144,7 +147,8 @@ class MovieDatabase: rss_updated = set() for torrent in torrent_list: match = title_regex.search(torrent.name) - if match: + release = parse_release_title(torrent.name) + if match and release is not None and release.media_type is MediaType.MOVIE: matched_title = match.group(0) match_data = title_index[matched_title] if ( diff --git a/backend/src/module/rss/analyser.py b/backend/src/module/rss/analyser.py index a6836120..0abd3dd5 100644 --- a/backend/src/module/rss/analyser.py +++ b/backend/src/module/rss/analyser.py @@ -109,20 +109,25 @@ class RSSAnalyser: ) -> tuple[list[Bangumi], list[Movie]]: new_bangumi: list[Bangumi] = [] new_movies: list[Movie] = [] - seen_titles: set[str] = set() + seen_identities: set[tuple[str, str, int]] = set() for torrent in torrents: result = await TitleParser.raw_parser(raw=torrent.name) if result is None or not result.title_raw: continue title_raw = result.title_raw - if title_raw in seen_titles: + identity = ( + ("movie", title_raw, 0) + if isinstance(result, Movie) + else (result.episode_type, title_raw, result.season) + ) + if identity in seen_identities: continue if isinstance(result, Movie): await self.official_title_parser_movie( movie=result, rss=rss, torrent=torrent ) result.rss_link = rss.url - seen_titles.add(title_raw) + seen_identities.add(identity) new_movies.append(result) logger.info(f"New movie found: {result.official_title}") elif isinstance(result, Bangumi): @@ -132,7 +137,7 @@ class RSSAnalyser: result.rss_link = rss.url if not full_parse: return [result], new_movies - seen_titles.add(title_raw) + seen_identities.add(identity) new_bangumi.append(result) logger.info(f"New bangumi found: {result.official_title}") return new_bangumi, new_movies diff --git a/backend/src/test/test_database.py b/backend/src/test/test_database.py index f2cc57dd..29749ca9 100644 --- a/backend/src/test/test_database.py +++ b/backend/src/test/test_database.py @@ -1,9 +1,10 @@ import json -from module.database.bangumi import BangumiDatabase +from module.database.bangumi import BangumiDatabase, match_bangumi_in_list +from module.database.movie import MovieDatabase from module.database.rss import RSSDatabase from module.database.torrent import TorrentDatabase -from module.models import Bangumi, RSSItem, Torrent +from module.models import Bangumi, Movie, RSSItem, Torrent async def _ensure_bangumi(session, bangumi_id: int): @@ -455,6 +456,35 @@ async def test_add_with_semantic_duplicate_creates_alias(db_session): assert "Frieren Beyond Journey's End" in aliases +async def test_typed_bangumi_identities_can_coexist_and_match(db_session): + db = BangumiDatabase(db_session) + shared = dict( + official_title="Shared Anime", + title_raw="Shared Anime", + group_name="Group", + dpi="1080p", + source="WEB-DL", + subtitle="CHT", + rss_link="rss", + ) + episode = Bangumi(**shared, season=1, episode_type="episode") + season_two = Bangumi(**shared, season=2, episode_type="episode") + special = Bangumi(**shared, season=0, episode_type="special") + + assert await db.add(episode) is True + assert await db.add(season_two) is True + assert await db.add(special) is True + + rows = await db.search_all() + assert len(rows) == 3 + matched_s2 = match_bangumi_in_list("[Group] Shared Anime S02E01 [1080p]", rows) + matched_ova = match_bangumi_in_list("[Group] Shared Anime OVA01 [1080p]", rows) + matched_episode = match_bangumi_in_list("[Group] Shared Anime - 01 [1080p]", rows) + assert matched_s2 is not None and matched_s2.season == 2 + assert matched_ova is not None and matched_ova.episode_type == "special" + assert matched_episode is not None and matched_episode.episode_type == "episode" + + async def test_add_same_title_new_entry_inherits_year(db_session): """A new entry for an already-known anime (same official_title, different group/quality so no semantic merge) must inherit the existing year, so both @@ -911,6 +941,44 @@ async def test_match_list_with_aliases(db_session): assert unmatched[0].name == "[OtherGroup] Different Anime - 01.mkv" +async def test_bangumi_match_list_does_not_consume_same_title_ova(db_session): + db = BangumiDatabase(db_session) + await db.add( + Bangumi( + official_title="Shared Anime", + title_raw="Shared Anime", + season=1, + episode_type="episode", + group_name="Group", + rss_link="rss1", + ) + ) + episode = Torrent(name="[Group] Shared Anime - 01 [1080p]", url="episode") + ova = Torrent(name="[Group] Shared Anime OVA01 [1080p]", url="ova") + + unmatched = await db.match_list([episode, ova], "rss2") + + assert [torrent.url for torrent in unmatched] == ["ova"] + + +async def test_movie_match_list_does_not_consume_same_title_episode(db_session): + db = MovieDatabase(db_session) + await db.add( + Movie( + official_title="Shared Anime", + title_raw="Shared Anime", + group_name="Group", + rss_link="rss1", + ) + ) + episode = Torrent(name="[Group] Shared Anime - 01 [1080p]", url="episode") + movie = Torrent(name="[Group] Shared Anime Movie [1080p]", url="movie") + + unmatched = await db.match_list([episode, movie], "rss2") + + assert [torrent.url for torrent in unmatched] == ["episode"] + + # ============================================================ # RSS Foreign Key Constraint Tests # ============================================================ diff --git a/backend/src/test/test_rss_analyser_typed_identity.py b/backend/src/test/test_rss_analyser_typed_identity.py new file mode 100644 index 00000000..dd87d170 --- /dev/null +++ b/backend/src/test/test_rss_analyser_typed_identity.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from module.conf import settings +from module.models import Movie, Torrent +from module.models.config import LLM +from module.rss.analyser import RSSAnalyser +from test.factories import make_rss_item + + +@pytest.mark.parametrize("reverse", (False, True)) +async def test_feed_keeps_episode_special_and_movie_with_same_title( + reverse: bool, +) -> None: + torrents = [ + Torrent( + name="[Group] Shared Anime - 01 [1080p WEB-DL]", + url="https://example.com/episode", + ), + Torrent( + name="[Group] Shared Anime OVA01 [1080p WEB-DL]", + url="https://example.com/ova", + ), + Torrent( + name="[Group] Shared Anime Movie [1080p WEB-DL]", + url="https://example.com/movie", + ), + ] + if reverse: + torrents.reverse() + + rss = make_rss_item(parser="none") + with patch.object(settings, "llm", LLM(enable=False)): + bangumi, movies = await RSSAnalyser().torrents_to_data(torrents, rss) + + assert {(item.episode_type, item.season) for item in bangumi} == { + ("episode", 1), + ("special", 0), + } + assert len(movies) == 1 + assert isinstance(movies[0], Movie) + assert {item.title_raw for item in bangumi} == {"Shared Anime"} + assert {item.title_raw for item in movies} == {"Shared Anime"}