nas/Auto_Bangumi
1
0
Fork 0
mirror of https://github.com/EstrellaXD/Auto_Bangumi.git synced 2026-10-31 04:34:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(rss): preserve typed media identities

Browse Source
This commit is contained in:
Estrella Pan
2026-07-11 17:59:01 +02:00
parent 3a61b47b14
commit d251cc5746
5 changed files with 226 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

145
backend/src/module/database/bangumi.py
View File

@@ -1,7 +1,7 @@
import json
import logging
import re
from typing import Optional
from typing import TYPE_CHECKING, Optional
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql import func
@@ -9,6 +9,9 @@ from sqlmodel import and_, delete, false, or_, select
from module.models import Bangumi, BangumiUpdate
if TYPE_CHECKING:
from module.parser.analyser.tokenizer.result import ParsedRelease
logger = logging.getLogger(__name__)
@@ -110,12 +113,16 @@ def _find_semantic_match(data: Bangumi, candidates: list[Bangumi]) -> Optional[B
is_exact_duplicate = (
candidate.title_raw == data.title_raw
and candidate.group_name == data.group_name
and candidate.season == data.season
and candidate.episode_type == data.episode_type
)
if is_exact_duplicate:
continue
is_semantic_match = (
candidate.dpi == data.dpi
candidate.season == data.season
and candidate.episode_type == data.episode_type
and candidate.dpi == data.dpi
and candidate.subtitle == data.subtitle
and candidate.source == data.source
and _groups_are_similar(candidate.group_name, data.group_name)
@@ -151,18 +158,51 @@ def match_bangumi_in_list(
(the job the old module-level TTL cache used to do). Returns the bangumi
with the longest matching pattern for specificity.
"""
# Lazy import avoids the database -> parser -> database import cycle during
# application startup while still giving matching the typed parser result.
from module.parser.analyser.tokenizer import parse_release_title
release = parse_release_title(torrent_name)
best_match: Optional[Bangumi] = None
best_match_len = 0
best_rank = (-1, -1, -1)
for bangumi in bangumi_list:
if bangumi.deleted:
if bangumi.deleted or not _release_matches_bangumi(release, bangumi):
continue
for pattern in _all_title_patterns(bangumi):
if pattern in torrent_name and len(pattern) > best_match_len:
if pattern not in torrent_name:
continue
group_match = int(
release is not None
and _groups_are_similar(release.group, bangumi.group_name)
)
rank = (len(pattern), group_match, bangumi.season)
if rank > best_rank:
best_match = bangumi
best_match_len = len(pattern)
best_rank = rank
return best_match
def _release_matches_bangumi(release: "ParsedRelease | None", bangumi: Bangumi) -> bool:
if release is None:
return True
from module.parser.analyser.tokenizer import MediaType
if release.media_type in {MediaType.PV, MediaType.OPENING, MediaType.ENDING}:
return False
if release.media_type is MediaType.MOVIE:
expected_type = "movie"
elif release.media_type in {MediaType.OVA, MediaType.OAD, MediaType.SPECIAL}:
expected_type = "special"
else:
expected_type = "episode"
if bangumi.episode_type != expected_type:
return False
if expected_type == "special":
return bangumi.season == 0
return release.season is None or bangumi.season == release.season
class BangumiDatabase:
def __init__(self, session: AsyncSession):
self.session = session
@@ -247,7 +287,7 @@ class BangumiDatabase:
return _all_title_patterns(bangumi)
async def find_duplicate(self, data: Bangumi) -> Optional[Bangumi]:
"""Find an existing bangumi with the same title_raw and group_name.
"""Find an existing rule with the same typed subscription identity.
Legacy data may contain multiple rows for the same key; return the
oldest (lowest id) deterministically and warn instead of raising.
@@ -258,6 +298,8 @@ class BangumiDatabase:
and_(
Bangumi.title_raw == data.title_raw,
Bangumi.group_name == data.group_name,
Bangumi.season == data.season,
Bangumi.episode_type == data.episode_type,
)
)
.order_by(Bangumi.id) # type: ignore[arg-type]
@@ -266,10 +308,13 @@ class BangumiDatabase:
rows = list(result.scalars().all())
if len(rows) > 1:
logger.warning(
"Multiple bangumi rows share (title_raw=%s, group_name=%s); "
"Multiple bangumi rows share (title_raw=%s, group_name=%s, "
"season=%s, episode_type=%s); "
"using the oldest (id=%s)",
data.title_raw,
data.group_name,
data.season,
data.episode_type,
rows[0].id,
)
return rows[0] if rows else None
@@ -310,24 +355,38 @@ class BangumiDatabase:
if not datas:
return 0
# Batch query: get all existing (title_raw, group_name) combinations in one query
# Batch query: load all existing typed subscription identities at once.
# This replaces N individual _is_duplicate() calls with a single SELECT
keys_to_check = [(d.title_raw, d.group_name) for d in datas]
keys_to_check = [
(d.title_raw, d.group_name, d.season, d.episode_type) for d in datas
]
conditions = [
and_(Bangumi.title_raw == tr, Bangumi.group_name == gn)
for tr, gn in keys_to_check
and_(
Bangumi.title_raw == tr,
Bangumi.group_name == gn,
Bangumi.season == season,
Bangumi.episode_type == episode_type,
)
for tr, gn, season, episode_type in keys_to_check
]
if conditions:
statement = select(Bangumi.title_raw, Bangumi.group_name).where(
or_(*conditions)
)
statement = select(
Bangumi.title_raw,
Bangumi.group_name,
Bangumi.season,
Bangumi.episode_type,
).where(or_(*conditions))
result = await self.session.execute(statement)
existing = set(result.all())
else:
existing = set()
# Filter out exact duplicates
to_add = [d for d in datas if (d.title_raw, d.group_name) not in existing]
to_add = [
d
for d in datas
if (d.title_raw, d.group_name, d.season, d.episode_type) not in existing
]
# Batch query: load all semantic-duplicate candidates (grouped by
# official_title) in one SELECT, instead of one find_semantic_duplicate()
@@ -376,7 +435,7 @@ class BangumiDatabase:
seen = set()
unique_to_add = []
for d in really_to_add:
key = (d.title_raw, d.group_name)
key = (d.title_raw, d.group_name, d.season, d.episode_type)
if key not in seen:
seen.add(key)
unique_to_add.append(d)
@@ -506,9 +565,17 @@ class BangumiDatabase:
return bangumi
async def search_official_title(self, official_title: str) -> Optional[Bangumi]:
statement = select(Bangumi).where(Bangumi.official_title == official_title)
statement = (
select(Bangumi)
.where(Bangumi.official_title == official_title)
.order_by(
Bangumi.episode_type, # type: ignore[arg-type]
Bangumi.season, # type: ignore[arg-type]
Bangumi.id, # type: ignore[arg-type]
)
)
result = await self.session.execute(statement)
return result.scalar_one_or_none()
return result.scalars().first()
async def search_ids(self, ids: list[int]) -> list[Bangumi]:
"""Batch lookup multiple bangumi by their IDs."""
@@ -531,41 +598,21 @@ class BangumiDatabase:
if not match_datas:
return torrent_list
# Build index for O(1) lookup after regex match
# Include both title_raw and all aliases
title_index: dict[str, Bangumi] = {}
for m in match_datas:
# Add main title_raw (skip if None to avoid TypeError in sorted())
if m.title_raw:
title_index[m.title_raw] = m
# Add all aliases
for alias in _get_aliases_list(m):
if alias:
title_index[alias] = m
# Build compiled regex pattern for fast substring matching
# Sort by length descending so longer (more specific) matches are found first
sorted_titles = sorted(title_index.keys(), key=len, reverse=True)
# Escape special regex characters and join with alternation
pattern = "|".join(re.escape(title) for title in sorted_titles)
title_regex = re.compile(pattern)
unmatched = []
rss_updated = set()
rss_updated: set[int | tuple[str, int, str]] = set()
for torrent in torrent_list:
match = title_regex.search(torrent.name)
if match:
matched_title = match.group(0)
match_data = title_index[matched_title]
# Use the bangumi's main title_raw for rss_updated tracking
if (
rss_link not in match_data.rss_link
and match_data.title_raw not in rss_updated
):
match_data = match_bangumi_in_list(torrent.name, match_datas)
if match_data is not None:
match_key: int | tuple[str, int, str] = match_data.id or (
match_data.title_raw,
match_data.season,
match_data.episode_type,
)
if rss_link not in match_data.rss_link and match_key not in rss_updated:
match_data = await self.session.merge(match_data)
match_data.rss_link += f",{rss_link}"
match_data.added = False
rss_updated.add(match_data.title_raw)
rss_updated.add(match_key)
else:
unmatched.append(torrent)
# Batch commit all rss_link updates

6
backend/src/module/database/movie.py
View File

@@ -124,6 +124,9 @@ class MovieDatabase:
logger.debug("Enable movie %s.", movie.title_raw)
async def match_list(self, torrent_list: list, rss_link: str) -> list:
# Lazy import avoids a database/parser import cycle during startup.
from module.parser.analyser.tokenizer import MediaType, parse_release_title
match_datas = await self.search_all()
if not match_datas:
return torrent_list
@@ -144,7 +147,8 @@ class MovieDatabase:
rss_updated = set()
for torrent in torrent_list:
match = title_regex.search(torrent.name)
if match:
release = parse_release_title(torrent.name)
if match and release is not None and release.media_type is MediaType.MOVIE:
matched_title = match.group(0)
match_data = title_index[matched_title]
if (

13
backend/src/module/rss/analyser.py
View File

@@ -109,20 +109,25 @@ class RSSAnalyser:
) -> tuple[list[Bangumi], list[Movie]]:
new_bangumi: list[Bangumi] = []
new_movies: list[Movie] = []
seen_titles: set[str] = set()
seen_identities: set[tuple[str, str, int]] = set()
for torrent in torrents:
result = await TitleParser.raw_parser(raw=torrent.name)
if result is None or not result.title_raw:
continue
title_raw = result.title_raw
if title_raw in seen_titles:
identity = (
("movie", title_raw, 0)
if isinstance(result, Movie)
else (result.episode_type, title_raw, result.season)
)
if identity in seen_identities:
continue
if isinstance(result, Movie):
await self.official_title_parser_movie(
movie=result, rss=rss, torrent=torrent
)
result.rss_link = rss.url
seen_titles.add(title_raw)
seen_identities.add(identity)
new_movies.append(result)
logger.info(f"New movie found: {result.official_title}")
elif isinstance(result, Bangumi):
@@ -132,7 +137,7 @@ class RSSAnalyser:
result.rss_link = rss.url
if not full_parse:
return [result], new_movies
seen_titles.add(title_raw)
seen_identities.add(identity)
new_bangumi.append(result)
logger.info(f"New bangumi found: {result.official_title}")
return new_bangumi, new_movies

72
backend/src/test/test_database.py
View File

@@ -1,9 +1,10 @@
import json
from module.database.bangumi import BangumiDatabase
from module.database.bangumi import BangumiDatabase, match_bangumi_in_list
from module.database.movie import MovieDatabase
from module.database.rss import RSSDatabase
from module.database.torrent import TorrentDatabase
from module.models import Bangumi, RSSItem, Torrent
from module.models import Bangumi, Movie, RSSItem, Torrent
async def _ensure_bangumi(session, bangumi_id: int):
@@ -455,6 +456,35 @@ async def test_add_with_semantic_duplicate_creates_alias(db_session):
assert "Frieren Beyond Journey's End" in aliases
async def test_typed_bangumi_identities_can_coexist_and_match(db_session):
db = BangumiDatabase(db_session)
shared = dict(
official_title="Shared Anime",
title_raw="Shared Anime",
group_name="Group",
dpi="1080p",
source="WEB-DL",
subtitle="CHT",
rss_link="rss",
)
episode = Bangumi(**shared, season=1, episode_type="episode")
season_two = Bangumi(**shared, season=2, episode_type="episode")
special = Bangumi(**shared, season=0, episode_type="special")
assert await db.add(episode) is True
assert await db.add(season_two) is True
assert await db.add(special) is True
rows = await db.search_all()
assert len(rows) == 3
matched_s2 = match_bangumi_in_list("[Group] Shared Anime S02E01 [1080p]", rows)
matched_ova = match_bangumi_in_list("[Group] Shared Anime OVA01 [1080p]", rows)
matched_episode = match_bangumi_in_list("[Group] Shared Anime - 01 [1080p]", rows)
assert matched_s2 is not None and matched_s2.season == 2
assert matched_ova is not None and matched_ova.episode_type == "special"
assert matched_episode is not None and matched_episode.episode_type == "episode"
async def test_add_same_title_new_entry_inherits_year(db_session):
"""A new entry for an already-known anime (same official_title, different
group/quality so no semantic merge) must inherit the existing year, so both
@@ -911,6 +941,44 @@ async def test_match_list_with_aliases(db_session):
assert unmatched[0].name == "[OtherGroup] Different Anime - 01.mkv"
async def test_bangumi_match_list_does_not_consume_same_title_ova(db_session):
db = BangumiDatabase(db_session)
await db.add(
Bangumi(
official_title="Shared Anime",
title_raw="Shared Anime",
season=1,
episode_type="episode",
group_name="Group",
rss_link="rss1",
)
)
episode = Torrent(name="[Group] Shared Anime - 01 [1080p]", url="episode")
ova = Torrent(name="[Group] Shared Anime OVA01 [1080p]", url="ova")
unmatched = await db.match_list([episode, ova], "rss2")
assert [torrent.url for torrent in unmatched] == ["ova"]
async def test_movie_match_list_does_not_consume_same_title_episode(db_session):
db = MovieDatabase(db_session)
await db.add(
Movie(
official_title="Shared Anime",
title_raw="Shared Anime",
group_name="Group",
rss_link="rss1",
)
)
episode = Torrent(name="[Group] Shared Anime - 01 [1080p]", url="episode")
movie = Torrent(name="[Group] Shared Anime Movie [1080p]", url="movie")
unmatched = await db.match_list([episode, movie], "rss2")
assert [torrent.url for torrent in unmatched] == ["episode"]
# ============================================================
# RSS Foreign Key Constraint Tests
# ============================================================

46
backend/src/test/test_rss_analyser_typed_identity.py Normal file
View File

@@ -0,0 +1,46 @@
from __future__ import annotations
from unittest.mock import patch
import pytest
from module.conf import settings
from module.models import Movie, Torrent
from module.models.config import LLM
from module.rss.analyser import RSSAnalyser
from test.factories import make_rss_item
@pytest.mark.parametrize("reverse", (False, True))
async def test_feed_keeps_episode_special_and_movie_with_same_title(
reverse: bool,
) -> None:
torrents = [
Torrent(
name="[Group] Shared Anime - 01 [1080p WEB-DL]",
url="https://example.com/episode",
),
Torrent(
name="[Group] Shared Anime OVA01 [1080p WEB-DL]",
url="https://example.com/ova",
),
Torrent(
name="[Group] Shared Anime Movie [1080p WEB-DL]",
url="https://example.com/movie",
),
]
if reverse:
torrents.reverse()
rss = make_rss_item(parser="none")
with patch.object(settings, "llm", LLM(enable=False)):
bangumi, movies = await RSSAnalyser().torrents_to_data(torrents, rss)
assert {(item.episode_type, item.season) for item in bangumi} == {
("episode", 1),
("special", 0),
}
assert len(movies) == 1
assert isinstance(movies[0], Movie)
assert {item.title_raw for item in bangumi} == {"Shared Anime"}
assert {item.title_raw for item in movies} == {"Shared Anime"}