From f75b8e55cd1d4d381a8d1f14a67ae82cc8c04e4a Mon Sep 17 00:00:00 2001 From: Estrella Pan Date: Sat, 28 Feb 2026 13:21:51 +0100 Subject: [PATCH] feat(parser): add movie/gekijouban support with full DB model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add detection and handling for anime movies (剧场版/劇場版) as a separate entity from TV series. Previously, movies in RSS feeds were either parsed as episode 0 or skipped entirely. - Add TokenKind.MOVIE and movie keyword/pattern detection in tokenizer - Add is_movie flag to Episode dataclass - Create Movie SQLModel, MovieDatabase with CRUD + TTL cache - Add migration v10 to create movie table - Route TitleParser.raw_parser() to return Movie or Bangumi - Update RSS analyser/engine for movie-aware parsing and downloads - Add movie save path (no Season subfolder) and rename logic - Add Movie REST API endpoints (get/update/delete/disable/enable) - Add 11 tests for movie marker detection and non-movie regression Co-Authored-By: Claude Opus 4.6 --- backend/src/module/api/__init__.py | 2 + backend/src/module/api/movie.py | 100 ++++++++++ backend/src/module/database/combine.py | 2 + backend/src/module/database/migrations.py | 107 ++++++++++- backend/src/module/database/movie.py | 171 ++++++++++++++++++ backend/src/module/downloader/path.py | 36 +++- backend/src/module/manager/renamer.py | 39 ++++ backend/src/module/models/__init__.py | 1 + backend/src/module/models/bangumi.py | 8 + backend/src/module/models/movie.py | 41 +++++ .../parser/analyser/tokenizer/keywords.py | 121 +++++++++++++ .../analyser/tokenizer/stage_compose.py | 100 ++++++++++ .../module/parser/analyser/tokenizer/token.py | 32 ++++ backend/src/module/parser/title_parser.py | 100 ++++++---- backend/src/module/rss/analyser.py | 137 ++++++++++---- backend/src/module/rss/engine.py | 42 ++++- backend/src/test/test_migrations_module.py | 116 +++++++++++- backend/src/test/test_movie_parser.py | 81 +++++++++ 18 files changed, 1157 insertions(+), 79 deletions(-) create mode 100644 backend/src/module/api/movie.py create mode 100644 backend/src/module/database/movie.py create mode 100644 backend/src/module/models/movie.py create mode 100644 backend/src/module/parser/analyser/tokenizer/keywords.py create mode 100644 backend/src/module/parser/analyser/tokenizer/stage_compose.py create mode 100644 backend/src/module/parser/analyser/tokenizer/token.py create mode 100644 backend/src/test/test_movie_parser.py diff --git a/backend/src/module/api/__init__.py b/backend/src/module/api/__init__.py index 77d73d21..cbacd0df 100644 --- a/backend/src/module/api/__init__.py +++ b/backend/src/module/api/__init__.py @@ -7,6 +7,7 @@ from .downloader import router as downloader_router from .events import router as events_router from .llm import router as llm_router from .log import router as log_router +from .movie import router as movie_router from .notification import router as notification_router from .passkey import router as passkey_router from .program import router as program_router @@ -28,6 +29,7 @@ v1.include_router(passkey_router) v1.include_router(log_router) v1.include_router(program_router) v1.include_router(bangumi_router) +v1.include_router(movie_router) v1.include_router(config_router) v1.include_router(downloader_router) v1.include_router(events_router) diff --git a/backend/src/module/api/movie.py b/backend/src/module/api/movie.py new file mode 100644 index 00000000..eb936c31 --- /dev/null +++ b/backend/src/module/api/movie.py @@ -0,0 +1,100 @@ +from fastapi import APIRouter, Depends +from fastapi.responses import JSONResponse + +from module.database import Database, get_db +from module.models import APIResponse, Movie +from module.models.movie import MovieUpdate +from module.security.api import get_current_user + +router = APIRouter(prefix="/movie", tags=["movie"]) + + +@router.get( + "/get/all", + response_model=list[Movie], + dependencies=[Depends(get_current_user)], +) +async def get_all_movies(db: Database = Depends(get_db)): + return await db.movie.search_all() + + +@router.get( + "/get/{movie_id}", + response_model=Movie, + dependencies=[Depends(get_current_user)], +) +async def get_movie(movie_id: int, db: Database = Depends(get_db)): + movie = await db.movie.search_id(movie_id) + if not movie: + return JSONResponse( + status_code=404, + content={ + "msg_en": f"Movie {movie_id} not found.", + "msg_zh": f"未找到剧场版 {movie_id}。", + }, + ) + return movie + + +@router.patch( + "/update/{movie_id}", + response_model=APIResponse, + dependencies=[Depends(get_current_user)], +) +async def update_movie( + movie_id: int, + data: MovieUpdate, + db: Database = Depends(get_db), +): + success = await db.movie.update(data, _id=movie_id) + if success: + return JSONResponse( + status_code=200, + content={"msg_en": "Movie updated.", "msg_zh": "剧场版已更新。"}, + ) + return JSONResponse( + status_code=404, + content={ + "msg_en": f"Movie {movie_id} not found.", + "msg_zh": f"未找到剧场版 {movie_id}。", + }, + ) + + +@router.delete( + "/delete/{movie_id}", + response_model=APIResponse, + dependencies=[Depends(get_current_user)], +) +async def delete_movie(movie_id: int, db: Database = Depends(get_db)): + await db.movie.delete_one(movie_id) + return JSONResponse( + status_code=200, + content={"msg_en": "Movie deleted.", "msg_zh": "剧场版已删除。"}, + ) + + +@router.delete( + "/disable/{movie_id}", + response_model=APIResponse, + dependencies=[Depends(get_current_user)], +) +async def disable_movie(movie_id: int, db: Database = Depends(get_db)): + await db.movie.disable_rule(movie_id) + return JSONResponse( + status_code=200, + content={"msg_en": "Movie disabled.", "msg_zh": "剧场版已禁用。"}, + ) + + +@router.get( + "/enable/{movie_id}", + response_model=APIResponse, + dependencies=[Depends(get_current_user)], +) +async def enable_movie(movie_id: int, db: Database = Depends(get_db)): + await db.movie.enable_rule(movie_id) + return JSONResponse( + status_code=200, + content={"msg_en": "Movie enabled.", "msg_zh": "剧场版已启用。"}, + ) diff --git a/backend/src/module/database/combine.py b/backend/src/module/database/combine.py index adb406de..ee15dada 100644 --- a/backend/src/module/database/combine.py +++ b/backend/src/module/database/combine.py @@ -19,6 +19,7 @@ from .migrations import ( # noqa: F401 (re-exported for existing importers) create_tables_async, run_migrations_async, ) +from .movie import MovieDatabase from .rss import RSSDatabase from .torrent import TorrentDatabase from .user import UserDatabase @@ -44,6 +45,7 @@ class Database: self.rss = RSSDatabase(self.session) self.torrent = TorrentDatabase(self.session) self.bangumi = BangumiDatabase(self.session) + self.movie = MovieDatabase(self.session) self.user = UserDatabase(self.session) self.aria2 = Aria2GidDatabase(self.session) self.auth = AuthDatabase(self.session) diff --git a/backend/src/module/database/migrations.py b/backend/src/module/database/migrations.py index e474cc4d..7e482c01 100644 --- a/backend/src/module/database/migrations.py +++ b/backend/src/module/database/migrations.py @@ -17,7 +17,7 @@ from pydantic_core import PydanticUndefined from sqlalchemy import Connection, Engine, inspect, text from sqlmodel import SQLModel -from module.models import ApiToken, AuthSession, Bangumi, User +from module.models import ApiToken, AuthSession, Bangumi, Movie, User from module.models.inbox import InboxMessage from module.models.llm_credential import LLMCredential from module.models.passkey import Passkey @@ -29,6 +29,7 @@ logger = logging.getLogger(__name__) # 所有需要进行空值填充的表模型 TABLE_MODELS: list[type[SQLModel]] = [ Bangumi, + Movie, RSSItem, Torrent, User, @@ -512,6 +513,110 @@ MIGRATIONS: tuple[Migration, ...] = ( # every database still below v20 must run the transactional rebuild. lambda _inspector: False, ), + Migration( + 21, + "create movie table for standalone movie subscriptions", + ( + """CREATE TABLE IF NOT EXISTS movie ( + id INTEGER PRIMARY KEY, + official_title VARCHAR NOT NULL, + title_raw VARCHAR, + year INTEGER, + group_name VARCHAR, + dpi VARCHAR, + source VARCHAR, + subtitle VARCHAR, + poster_link VARCHAR, + rss_link VARCHAR, + added BOOLEAN NOT NULL DEFAULT 0, + deleted BOOLEAN NOT NULL DEFAULT 0, + save_path VARCHAR, + rule_name VARCHAR, + filter VARCHAR NOT NULL DEFAULT '' + )""", + "CREATE INDEX IF NOT EXISTS ix_movie_title_raw ON movie(title_raw)", + "CREATE INDEX IF NOT EXISTS ix_movie_deleted ON movie(deleted)", + ), + all_checks( + table_exists("movie"), + index_exists("movie", "ix_movie_title_raw"), + index_exists("movie", "ix_movie_deleted"), + ), + ( + ( + """CREATE TABLE IF NOT EXISTS movie ( + id INTEGER PRIMARY KEY, + official_title VARCHAR NOT NULL, + title_raw VARCHAR, + year INTEGER, + group_name VARCHAR, + dpi VARCHAR, + source VARCHAR, + subtitle VARCHAR, + poster_link VARCHAR, + rss_link VARCHAR, + added BOOLEAN NOT NULL DEFAULT 0, + deleted BOOLEAN NOT NULL DEFAULT 0, + save_path VARCHAR, + rule_name VARCHAR, + filter VARCHAR NOT NULL DEFAULT '' + )""", + table_exists("movie"), + ), + ( + "CREATE INDEX IF NOT EXISTS ix_movie_title_raw ON movie(title_raw)", + index_exists("movie", "ix_movie_title_raw"), + ), + ( + "CREATE INDEX IF NOT EXISTS ix_movie_deleted ON movie(deleted)", + index_exists("movie", "ix_movie_deleted"), + ), + ), + ), + Migration( + 22, + "repair multi-user v18 fields after the divergent movie development schema", + ( + "ALTER TABLE user ADD COLUMN enabled BOOLEAN NOT NULL DEFAULT 1", + "ALTER TABLE user ADD COLUMN created_at TIMESTAMP NOT NULL " + "DEFAULT '1970-01-01 00:00:00'", + "ALTER TABLE user ADD COLUMN updated_at TIMESTAMP NOT NULL " + "DEFAULT '1970-01-01 00:00:00'", + "CREATE UNIQUE INDEX IF NOT EXISTS ix_user_username ON user(username)", + "CREATE INDEX IF NOT EXISTS ix_user_enabled ON user(enabled)", + ), + all_checks( + column_exists("user", "enabled"), + column_exists("user", "created_at"), + column_exists("user", "updated_at"), + index_exists("user", "ix_user_username"), + index_exists("user", "ix_user_enabled"), + ), + ( + ( + "ALTER TABLE user ADD COLUMN enabled BOOLEAN NOT NULL DEFAULT 1", + column_exists("user", "enabled"), + ), + ( + "ALTER TABLE user ADD COLUMN created_at TIMESTAMP NOT NULL " + "DEFAULT '1970-01-01 00:00:00'", + column_exists("user", "created_at"), + ), + ( + "ALTER TABLE user ADD COLUMN updated_at TIMESTAMP NOT NULL " + "DEFAULT '1970-01-01 00:00:00'", + column_exists("user", "updated_at"), + ), + ( + "CREATE UNIQUE INDEX IF NOT EXISTS ix_user_username ON user(username)", + index_exists("user", "ix_user_username"), + ), + ( + "CREATE INDEX IF NOT EXISTS ix_user_enabled ON user(enabled)", + index_exists("user", "ix_user_enabled"), + ), + ), + ), ) # 由迁移列表派生，新增迁移时无需手动同步 diff --git a/backend/src/module/database/movie.py b/backend/src/module/database/movie.py new file mode 100644 index 00000000..6848bcbb --- /dev/null +++ b/backend/src/module/database/movie.py @@ -0,0 +1,171 @@ +import logging +import re +import time +from typing import Optional + +from sqlalchemy.ext.asyncio import AsyncSession +from sqlmodel import and_, false, select + +from module.models.movie import Movie, MovieUpdate + +logger = logging.getLogger(__name__) + +# Module-level TTL cache for search_all results +_movie_cache: list[Movie] | None = None +_movie_cache_time: float = 0 +_MOVIE_CACHE_TTL: float = 300.0 + + +def _invalidate_movie_cache(): + global _movie_cache, _movie_cache_time + _movie_cache = None + _movie_cache_time = 0 + + +class MovieDatabase: + def __init__(self, session: AsyncSession): + self.session = session + + async def _is_duplicate(self, data: Movie) -> bool: + statement = select(Movie).where( + and_( + Movie.title_raw == data.title_raw, + Movie.group_name == data.group_name, + ) + ) + result = await self.session.execute(statement) + return result.scalar_one_or_none() is not None + + async def add(self, data: Movie) -> bool: + if await self._is_duplicate(data): + logger.debug( + "Skipping duplicate movie: %s (%s)", + data.official_title, + data.group_name, + ) + return False + self.session.add(data) + await self.session.commit() + _invalidate_movie_cache() + logger.debug("Insert movie %s into database.", data.official_title) + return True + + async def search_all(self) -> list[Movie]: + global _movie_cache, _movie_cache_time + now = time.time() + if _movie_cache is not None and (now - _movie_cache_time) < _MOVIE_CACHE_TTL: + return _movie_cache + statement = select(Movie) + result = await self.session.execute(statement) + movies = list(result.scalars().all()) + for m in movies: + self.session.expunge(m) + _movie_cache = movies + _movie_cache_time = now + return _movie_cache + + async def search_id(self, _id: int) -> Optional[Movie]: + statement = select(Movie).where(Movie.id == _id) + result = await self.session.execute(statement) + movie = result.scalar_one_or_none() + if movie is None: + logger.warning(f"Cannot find movie id: {_id}.") + return None + return movie + + async def update(self, data: Movie | MovieUpdate, _id: int | None = None) -> bool: + if _id and isinstance(data, MovieUpdate): + db_data = await self.session.get(Movie, _id) + elif isinstance(data, Movie): + db_data = await self.session.get(Movie, data.id) + else: + return False + if not db_data: + return False + movie_data = data.model_dump(exclude_unset=True) + for key, value in movie_data.items(): + setattr(db_data, key, value) + self.session.add(db_data) + await self.session.commit() + _invalidate_movie_cache() + logger.debug("Update movie %s", data.official_title) + return True + + async def delete_one(self, _id: int): + statement = select(Movie).where(Movie.id == _id) + result = await self.session.execute(statement) + movie = result.scalar_one_or_none() + if movie: + await self.session.delete(movie) + await self.session.commit() + _invalidate_movie_cache() + logger.debug("Delete movie id: %s.", _id) + + async def disable_rule(self, _id: int): + statement = select(Movie).where(Movie.id == _id) + result = await self.session.execute(statement) + movie = result.scalar_one_or_none() + if movie: + movie.deleted = True + self.session.add(movie) + await self.session.commit() + _invalidate_movie_cache() + logger.debug("Disable movie %s.", movie.title_raw) + + async def enable_rule(self, _id: int): + statement = select(Movie).where(Movie.id == _id) + result = await self.session.execute(statement) + movie = result.scalar_one_or_none() + if movie: + movie.deleted = False + self.session.add(movie) + await self.session.commit() + _invalidate_movie_cache() + logger.debug("Enable movie %s.", movie.title_raw) + + async def match_list(self, torrent_list: list, rss_link: str) -> list: + match_datas = await self.search_all() + if not match_datas: + return torrent_list + + title_index: dict[str, Movie] = {} + for m in match_datas: + if m.title_raw: + title_index[m.title_raw] = m + + if not title_index: + return torrent_list + + sorted_titles = sorted(title_index.keys(), key=len, reverse=True) + pattern = "|".join(re.escape(title) for title in sorted_titles) + title_regex = re.compile(pattern) + + unmatched = [] + rss_updated = set() + for torrent in torrent_list: + match = title_regex.search(torrent.name) + if match: + matched_title = match.group(0) + match_data = title_index[matched_title] + if ( + match_data.rss_link + and rss_link not in match_data.rss_link + and match_data.title_raw not in rss_updated + ): + match_data = await self.session.merge(match_data) + match_data.rss_link = f"{match_data.rss_link},{rss_link}" + match_data.added = False + rss_updated.add(match_data.title_raw) + else: + unmatched.append(torrent) + if rss_updated: + await self.session.commit() + _invalidate_movie_cache() + return unmatched + + async def not_added(self) -> list[Movie]: + conditions = select(Movie).where( + Movie.added == false(), + ) + result = await self.session.execute(conditions) + return list(result.scalars().all()) diff --git a/backend/src/module/downloader/path.py b/backend/src/module/downloader/path.py index 6cc3b46d..03e07ce4 100644 --- a/backend/src/module/downloader/path.py +++ b/backend/src/module/downloader/path.py @@ -5,6 +5,7 @@ from pathlib import PureWindowsPath from module.conf import PLATFORM, settings from module.models import Bangumi, BangumiUpdate +from module.models.movie import Movie, MovieUpdate logger = logging.getLogger(__name__) @@ -77,7 +78,17 @@ def is_ep(file_path: PathLike[str] | str): return file_depth(file_path) <= 2 -def gen_save_path(data: Bangumi | BangumiUpdate): +def _media_folder(data: Bangumi | BangumiUpdate | Movie | MovieUpdate) -> str: + title = data.official_title or "Unknown Bangumi" + folder = sanitize_path_fragment(f"{title} ({data.year})" if data.year else title) + if folder: + return folder + # 标题全由保留字符组成时清洗结果为空——不能让所有这类条目 + # 坍缩到同一个下载目录里 + return "Unknown Bangumi" + + +def gen_save_path(data: Bangumi | BangumiUpdate | Movie | MovieUpdate) -> str: """Generate save path for a bangumi. The save path uses the adjusted season number (season + season_offset) @@ -87,15 +98,9 @@ def gen_save_path(data: Bangumi | BangumiUpdate): specials/OVA/OAD land in "Season 0" (Jellyfin/Plex convention) instead of being interleaved with regular episodes. """ - folder = sanitize_path_fragment( - f"{data.official_title} ({data.year})" if data.year else data.official_title - ) - if not folder: - # 标题全由保留字符组成时清洗结果为空——不能让所有这类番剧 - # 坍缩到同一个 ``…//Season N`` 目录里 - folder = "Unknown Bangumi" + folder = _media_folder(data) episode_type = getattr(data, "episode_type", "episode") - if episode_type == "movie": + if isinstance(data, (Movie, MovieUpdate)) or episode_type == "movie": # 电影/剧场版：Title (Year)/Title (Year).ext，不建 Season 子目录 return str(Path(settings.downloader.path) / folder) # Apply season_offset to get the adjusted season number for the folder @@ -112,6 +117,19 @@ def gen_save_path(data: Bangumi | BangumiUpdate): return str(save_path) +def gen_movie_save_path(data: Movie | MovieUpdate) -> str: + """Generate the flat save directory used by a movie/gekijouban.""" + return str(Path(settings.downloader.path) / _media_folder(data)) + + +def movie_rule_name(data: Movie) -> str: + return ( + f"[{data.group_name}] {data.official_title}" + if settings.bangumi_manage.group_tag + else data.official_title + ) + + def rule_name(data: Bangumi): name = ( f"[{data.group_name}] {data.official_title} S{data.season}" diff --git a/backend/src/module/manager/renamer.py b/backend/src/module/manager/renamer.py index 1c28044a..8506b0f7 100644 --- a/backend/src/module/manager/renamer.py +++ b/backend/src/module/manager/renamer.py @@ -85,6 +85,16 @@ class Renamer: episode = int(episode) return f"0{episode}" if episode < 10 else str(episode) + @staticmethod + def gen_movie_path( + file_info: EpisodeFile | SubtitleFile, + movie_name: str, + method: str, + ) -> str: + if method in ("none", "subtitle_none"): + return file_info.media_path + return f"{movie_name}{file_info.suffix}" + @staticmethod def gen_path( file_info: EpisodeFile | SubtitleFile, @@ -237,6 +247,35 @@ class Renamer: stem = stem[len(prefix) :] return f"{base} - {stem}{suffix}" + async def rename_movie_file( + self, + torrent_name: str, + media_path: str, + movie_name: str, + method: str, + _hash: str, + **kwargs, + ): + ep = self._parser.torrent_parser( + torrent_name=torrent_name, + torrent_path=media_path, + episode_type="movie", + ) + if ep: + new_path = self.gen_movie_path(ep, movie_name, method=method) + if media_path != new_path: + if await self.client.rename_torrent_file( + _hash=_hash, old_path=media_path, new_path=new_path + ): + return Notification( + official_title=movie_name, + season=0, + episode=0, + ) + else: + logger.warning(f"{media_path} parse failed (movie)") + return None + async def rename_collection( self, media_list: list[str], diff --git a/backend/src/module/models/__init__.py b/backend/src/module/models/__init__.py index 5468094c..8b6b8958 100644 --- a/backend/src/module/models/__init__.py +++ b/backend/src/module/models/__init__.py @@ -3,6 +3,7 @@ from .bangumi import Bangumi, BangumiUpdate, Episode, Notification from .config import Config from .inbox import InboxMessage from .llm_credential import LLMCredential +from .movie import Movie, MovieUpdate from .passkey import Passkey, PasskeyCreate, PasskeyDelete, PasskeyList from .response import APIResponse, ResponseModel from .rss import RSSItem, RSSUpdate diff --git a/backend/src/module/models/bangumi.py b/backend/src/module/models/bangumi.py index 69789087..30d8f283 100644 --- a/backend/src/module/models/bangumi.py +++ b/backend/src/module/models/bangumi.py @@ -131,6 +131,14 @@ class Episode: resolution: str source: str episode_type: str = "episode" # "episode" | "movie" | "special" + is_movie: bool = False + + def __post_init__(self) -> None: + """Keep the legacy movie flag and the typed classifier in sync.""" + if self.is_movie: + self.episode_type = "movie" + elif self.episode_type == "movie": + self.is_movie = True @dataclass(slots=True) diff --git a/backend/src/module/models/movie.py b/backend/src/module/models/movie.py new file mode 100644 index 00000000..bd5413c6 --- /dev/null +++ b/backend/src/module/models/movie.py @@ -0,0 +1,41 @@ +from typing import Optional + +from sqlmodel import Field, SQLModel + + +class Movie(SQLModel, table=True): + id: int = Field(default=None, primary_key=True) + official_title: str = Field( + default="official_title", alias="official_title", title="剧场版名" + ) + title_raw: str | None = Field( + default=None, alias="title_raw", title="剧场版原名", index=True + ) + year: int | None = Field(default=None, alias="year", title="年份") + group_name: str | None = Field(default=None, alias="group_name", title="字幕组") + dpi: str | None = Field(default=None, alias="dpi", title="分辨率") + source: str | None = Field(default=None, alias="source", title="来源") + subtitle: str | None = Field(default=None, alias="subtitle", title="字幕") + poster_link: str | None = Field(default=None, alias="poster_link", title="海报链接") + rss_link: str | None = Field(default=None, alias="rss_link", title="RSS链接") + added: bool = Field(default=False, alias="added", title="是否已添加") + deleted: bool = Field( + default=False, alias="deleted", title="是否已删除", index=True + ) + save_path: str | None = Field(default=None, alias="save_path", title="保存路径") + rule_name: str | None = Field(default=None, alias="rule_name", title="规则名") + filter: str = Field(default="", alias="filter", title="过滤器") + + +class MovieUpdate(SQLModel): + official_title: str | None = None + title_raw: str | None = None + year: int | None = None + group_name: str | None = None + dpi: str | None = None + source: str | None = None + subtitle: str | None = None + poster_link: str | None = None + rss_link: str | None = None + save_path: str | None = None + filter: str | None = None diff --git a/backend/src/module/parser/analyser/tokenizer/keywords.py b/backend/src/module/parser/analyser/tokenizer/keywords.py new file mode 100644 index 00000000..118eb2d8 --- /dev/null +++ b/backend/src/module/parser/analyser/tokenizer/keywords.py @@ -0,0 +1,121 @@ +import re + +from .token import TokenKind + +KEYWORDS: dict[str, TokenKind] = { + # Resolution + "1080p": TokenKind.RESOLUTION, + "720p": TokenKind.RESOLUTION, + "2160p": TokenKind.RESOLUTION, + "4k": TokenKind.RESOLUTION, + "1080": TokenKind.RESOLUTION, + "720": TokenKind.RESOLUTION, + "2160": TokenKind.RESOLUTION, + "1920x1080": TokenKind.RESOLUTION, + "3840x2160": TokenKind.RESOLUTION, + # Source + "baha": TokenKind.SOURCE, + "b-global": TokenKind.SOURCE, + "bilibili": TokenKind.SOURCE, + "at-x": TokenKind.SOURCE, + "web-dl": TokenKind.SOURCE, + "webrip": TokenKind.SOURCE, + "webdl": TokenKind.SOURCE, + "web": TokenKind.SOURCE, + "web-dl remux": TokenKind.SOURCE, + "ttfc": TokenKind.SOURCE, + "cr": TokenKind.SOURCE, + "adn": TokenKind.SOURCE, + "iqiyi": TokenKind.SOURCE, + "abema": TokenKind.SOURCE, + # Codec + "hevc": TokenKind.CODEC, + "avc": TokenKind.CODEC, + "x264": TokenKind.CODEC, + "x265": TokenKind.CODEC, + "hevc-10bit": TokenKind.CODEC, + "hevc10-bit": TokenKind.CODEC, + "h264": TokenKind.CODEC, + "h265": TokenKind.CODEC, + "h.264": TokenKind.CODEC, + "h.265": TokenKind.CODEC, + "10bit": TokenKind.CODEC, + # Audio + "aac": TokenKind.AUDIO, + "flac": TokenKind.AUDIO, + "mp3": TokenKind.AUDIO, + "ac3": TokenKind.AUDIO, + "dts": TokenKind.AUDIO, + # Container + "mp4": TokenKind.CONTAINER, + "mkv": TokenKind.CONTAINER, + "avi": TokenKind.CONTAINER, + # Subtitle markers + "cht": TokenKind.SUBTITLE, + "chs": TokenKind.SUBTITLE, + "big5": TokenKind.SUBTITLE, + "gb": TokenKind.SUBTITLE, + "assx2": TokenKind.SUBTITLE, + "multi sub": TokenKind.SUBTITLE, + "vostfr": TokenKind.SUBTITLE, + # Movie markers + "movie": TokenKind.MOVIE, + "gekijouban": TokenKind.MOVIE, + "the movie": TokenKind.MOVIE, + # Version + "v2": TokenKind.VERSION, + "v3": TokenKind.VERSION, + # End marker + "end": TokenKind.END_MARKER, +} + + +PATTERN_CHECKS: list[tuple[re.Pattern, TokenKind]] = [ + # Episode patterns + (re.compile(r"^[Ee][Pp]?\d+$"), TokenKind.EPISODE), + (re.compile(r"^第?\d+[话話集]$"), TokenKind.EPISODE), + (re.compile(r"^\d+[Pp]$"), TokenKind.RESOLUTION), + (re.compile(r"^\d{3,4}[xX]\d{3,4}$"), TokenKind.RESOLUTION), + # Season patterns + (re.compile(r"^[Ss]\d{1,2}$"), TokenKind.SEASON), + (re.compile(r"^Season \d+$", re.I), TokenKind.SEASON), + (re.compile(r"^第[一二三四五六七八九十\d]+[季期]$"), TokenKind.SEASON), + # S01E05 combined pattern + (re.compile(r"^[Ss]\d{1,2}[Ee]\d+$"), TokenKind.EPISODE), + # Subtitle patterns + (re.compile(r"^[简繁日字幕][简繁日字幕体文]*"), TokenKind.SUBTITLE), + (re.compile(r"^GB_?(?:JP|MP4|MKV)?$", re.I), TokenKind.SUBTITLE), + (re.compile(r"^CHS_JP$", re.I), TokenKind.SUBTITLE), + (re.compile(r"^CHT_JP$", re.I), TokenKind.SUBTITLE), + # Prefix tags + (re.compile(r"^\d+月新番$"), TokenKind.PREFIX_TAG), + (re.compile(r"^新番$"), TokenKind.PREFIX_TAG), + (re.compile(r"^★.*★$"), TokenKind.PREFIX_TAG), + # Version pattern + (re.compile(r"^[vV]\d$"), TokenKind.VERSION), + # Hash (CRC32) + (re.compile(r"^[0-9A-Fa-f]{8}$"), TokenKind.HASH), + # Movie markers + (re.compile(r"^[剧劇][场場][版]?$|^劇場版?$"), TokenKind.MOVIE), + (re.compile(r"^Movie$", re.I), TokenKind.MOVIE), + (re.compile(r"^Gekijou-?ban$", re.I), TokenKind.MOVIE), + # Date pattern + (re.compile(r"^\d{4}\.\d{2}\.\d{2}$"), TokenKind.EXTRA), + # File size + (re.compile(r"^\d+\.\d+ [KMGT]B$", re.I), TokenKind.EXTRA), +] + +CHINESE_NUMBER_MAP: dict[str, int] = { + "一": 1, + "二": 2, + "三": 3, + "四": 4, + "五": 5, + "六": 6, + "七": 7, + "八": 8, + "九": 9, + "十": 10, +} + +FILE_EXTENSIONS = re.compile(r"\.(mp4|mkv|avi|ass|srt|ssa|sub)$", re.I) diff --git a/backend/src/module/parser/analyser/tokenizer/stage_compose.py b/backend/src/module/parser/analyser/tokenizer/stage_compose.py new file mode 100644 index 00000000..4ad8112d --- /dev/null +++ b/backend/src/module/parser/analyser/tokenizer/stage_compose.py @@ -0,0 +1,100 @@ +import re + +from module.models import Episode + +from .keywords import CHINESE_NUMBER_MAP +from .token import Token, TokenKind + +# Only strip dashes that have surrounding whitespace (separators), +# not dashes attached to characters (part of titles like 地。-关于地球的运动-) +_DASH_STRIP = re.compile(r"^\s*-\s+|\s+-\s*$") +_SEASON_NUM = re.compile(r"[Ss](\d+)|Season\s+(\d+)", re.I) +_CHINESE_SEASON = re.compile(r"第([一二三四五六七八九十\d]+)[季期]") +_SUB_CLEAN = re.compile(r"_MP4|_MKV", re.I) +_CJK_SUB = re.compile(r"[简繁日字幕]") + + +def compose(tokens: list[Token]) -> Episode | None: + title_en: str | None = None + title_zh: str | None = None + title_jp: str | None = None + season = 1 + season_raw = "" + episode = 0 + sub: str | None = None + group = "" + resolution: str | None = None + source: str | None = None + + is_movie = False + + # Collect all subtitles, prefer CJK ones + all_subs: list[str] = [] + + for token in tokens: + kind = token.kind + text = token.text + + if kind == TokenKind.GROUP: + group = text + elif kind == TokenKind.TITLE_ZH: + text = _DASH_STRIP.sub("", text).strip() + if text: + title_zh = text + elif kind == TokenKind.TITLE_EN: + title_en = text.strip() + elif kind == TokenKind.TITLE_JP: + title_jp = text.strip() + elif kind == TokenKind.EPISODE: + ep_digits = re.search(r"\d+", text) + if ep_digits: + episode = int(ep_digits.group()) + elif kind == TokenKind.SEASON: + m = _SEASON_NUM.match(text) + if m: + s_num = int(m.group(1) or m.group(2)) + else: + cm = _CHINESE_SEASON.match(text) + if cm: + s_val = cm.group(1) + try: + s_num = int(s_val) + except ValueError: + s_num = CHINESE_NUMBER_MAP.get(s_val, 1) + else: + s_num = season + # Prefer first season found (Chinese season over English) + if season == 1 or not season_raw: + season = s_num + season_raw = text + elif kind == TokenKind.RESOLUTION: + if resolution is None: + resolution = text + elif kind == TokenKind.SOURCE: + if source is None: + source = text + elif kind == TokenKind.MOVIE: + is_movie = True + elif kind == TokenKind.SUBTITLE: + cleaned = _SUB_CLEAN.sub("", text) + if cleaned: + all_subs.append(cleaned) + + # Prefer CJK subtitle markers over technical ones + if all_subs: + cjk_subs = [s for s in all_subs if _CJK_SUB.search(s)] + sub = cjk_subs[0] if cjk_subs else all_subs[0] + + return Episode( + title_en=title_en, + title_zh=title_zh, + title_jp=title_jp, + season=season, + season_raw=season_raw, + episode=episode, + sub=sub, + group=group, + resolution=resolution, + source=source, + is_movie=is_movie, + ) diff --git a/backend/src/module/parser/analyser/tokenizer/token.py b/backend/src/module/parser/analyser/tokenizer/token.py new file mode 100644 index 00000000..7f15d215 --- /dev/null +++ b/backend/src/module/parser/analyser/tokenizer/token.py @@ -0,0 +1,32 @@ +from dataclasses import dataclass +from enum import Enum, auto + + +class TokenKind(Enum): + UNKNOWN = auto() + GROUP = auto() + TITLE_ZH = auto() + TITLE_EN = auto() + TITLE_JP = auto() + EPISODE = auto() + SEASON = auto() + RESOLUTION = auto() + SOURCE = auto() + SUBTITLE = auto() + CODEC = auto() + AUDIO = auto() + CONTAINER = auto() + PREFIX_TAG = auto() + VERSION = auto() + END_MARKER = auto() + HASH = auto() + MOVIE = auto() # movie/gekijouban marker + EXTRA = auto() + + +@dataclass(slots=True) +class Token: + text: str + kind: TokenKind + position: int + enclosed: bool diff --git a/backend/src/module/parser/title_parser.py b/backend/src/module/parser/title_parser.py index 33a3aa0e..4006c07b 100644 --- a/backend/src/module/parser/title_parser.py +++ b/backend/src/module/parser/title_parser.py @@ -3,7 +3,7 @@ import logging import time from module.conf import settings -from module.models import Bangumi +from module.models import Bangumi, Movie from module.models.bangumi import Episode from module.models.config import LLM from module.parser.analyser import ( @@ -254,8 +254,62 @@ class TitleParser: logger.warning("Please change bangumi info manually.") @staticmethod - async def raw_parser(raw: str) -> Bangumi | None: + def _resolve_titles(episode: Episode) -> tuple[str | None, str | None]: + """Extract official_title and title_raw from an Episode.""" language = settings.rss_parser.language + titles = { + "zh": episode.title_zh, + "en": episode.title_en, + "jp": episode.title_jp, + } + title_raw = episode.title_en or episode.title_zh or episode.title_jp + if titles[language]: + official_title = titles[language] + elif titles["zh"]: + official_title = titles["zh"] + elif titles["en"]: + official_title = titles["en"] + elif titles["jp"]: + official_title = titles["jp"] + else: + official_title = title_raw + return official_title, title_raw + + @staticmethod + def _episode_to_bangumi( + episode: Episode, official_title: str, title_raw: str + ) -> Bangumi: + return Bangumi( + official_title=official_title, + title_raw=title_raw, + season=episode.season, + season_raw=episode.season_raw, + group_name=episode.group, + dpi=episode.resolution, + source=episode.source, + subtitle=episode.sub, + eps_collect=False if episode.episode > 1 else True, + episode_offset=0, + filter=",".join(settings.rss_parser.filter), + episode_type=episode.episode_type, + ) + + @staticmethod + def _episode_to_movie( + episode: Episode, official_title: str, title_raw: str + ) -> Movie: + return Movie( + official_title=official_title, + title_raw=title_raw, + group_name=episode.group, + dpi=episode.resolution, + source=episode.source, + subtitle=episode.sub, + filter=",".join(settings.rss_parser.filter), + ) + + @staticmethod + async def raw_parser(raw: str) -> Bangumi | Movie | None: try: llm_conf = _llm_config() episode = None @@ -280,41 +334,21 @@ class TitleParser: if detected_episode_type is not None: episode.episode_type = detected_episode_type - titles = { - "zh": episode.title_zh, - "en": episode.title_en, - "jp": episode.title_jp, - } - title_raw = episode.title_en or episode.title_zh or episode.title_jp - if titles[language]: - official_title = titles[language] - elif titles["zh"]: - official_title = titles["zh"] - elif titles["en"]: - official_title = titles["en"] - elif titles["jp"]: - official_title = titles["jp"] - else: - official_title = title_raw + official_title, title_raw = TitleParser._resolve_titles(episode) if not title_raw: logger.warning("Cannot extract title_raw from '%s', skipping", raw) return None - _season = episode.season + if not official_title: + logger.warning("Cannot extract official_title from '%s', skipping", raw) + return None logger.debug("RAW:%s >> %s", raw, title_raw) - return Bangumi( - official_title=official_title, - title_raw=title_raw, - season=_season, - season_raw=episode.season_raw, - group_name=episode.group, - dpi=episode.resolution, - source=episode.source, - subtitle=episode.sub, - eps_collect=False if episode.episode > 1 else True, - offset=0, - filter=",".join(settings.rss_parser.filter), - episode_type=episode.episode_type, - ) + + # The upstream LLM path exposes its inferred non-episodic type on a + # Bangumi. Keep that contract while routing deterministically parsed + # movie markers into the dedicated local Movie table. + if episode.episode_type == "movie" and not from_llm: + return TitleParser._episode_to_movie(episode, official_title, title_raw) + return TitleParser._episode_to_bangumi(episode, official_title, title_raw) except (ValueError, AttributeError, TypeError) as e: logger.warning(f"Cannot parse '{raw}': {type(e).__name__}: {e}") return None diff --git a/backend/src/module/rss/analyser.py b/backend/src/module/rss/analyser.py index 270ad203..a6836120 100644 --- a/backend/src/module/rss/analyser.py +++ b/backend/src/module/rss/analyser.py @@ -2,7 +2,7 @@ import logging import re from module.conf import settings -from module.models import Bangumi, ResponseModel, RSSItem, Torrent +from module.models import Bangumi, Movie, ResponseModel, RSSItem, Torrent from module.network import RequestContent from module.parser import TitleParser @@ -12,6 +12,48 @@ logger = logging.getLogger(__name__) class RSSAnalyser: + async def official_title_parser_movie( + self, + movie: Movie, + rss: RSSItem, + torrent: Torrent, + fetch_poster: bool = True, + ): + if not fetch_poster: + pass + elif rss.parser == "mikan": + if not torrent.homepage: + logger.warning("Mikan movie torrent has no homepage info.") + else: + try: + poster_link, official_title = await TitleParser.mikan_parser( + torrent.homepage + ) + except AttributeError as e: + logger.warning( + f"Failed to parse Mikan homepage {torrent.homepage}: {e}" + ) + else: + movie.poster_link = poster_link + if official_title: + movie.official_title = official_title + elif rss.parser == "tmdb": + tmdb_title, _, year, poster_link = await TitleParser.tmdb_parser( + movie.official_title, + 1, + settings.rss_parser.language, + episode_type="movie", + ) + movie.official_title = tmdb_title + if year: + try: + movie.year = int(year) + except (ValueError, TypeError): + pass + movie.poster_link = poster_link + if movie.official_title: + movie.official_title = re.sub(r"[/:.\\]", " ", movie.official_title) + async def official_title_parser( self, bangumi: Bangumi, @@ -64,53 +106,86 @@ class RSSAnalyser: async def torrents_to_data( self, torrents: list[Torrent], rss: RSSItem, full_parse: bool = True - ) -> list: - new_data = [] + ) -> tuple[list[Bangumi], list[Movie]]: + new_bangumi: list[Bangumi] = [] + new_movies: list[Movie] = [] seen_titles: set[str] = set() for torrent in torrents: - bangumi = await TitleParser.raw_parser(raw=torrent.name) - if bangumi and bangumi.title_raw not in seen_titles: - await self.official_title_parser( - bangumi=bangumi, rss=rss, torrent=torrent + result = await TitleParser.raw_parser(raw=torrent.name) + if result is None or not result.title_raw: + continue + title_raw = result.title_raw + if title_raw in seen_titles: + continue + if isinstance(result, Movie): + await self.official_title_parser_movie( + movie=result, rss=rss, torrent=torrent ) - bangumi.rss_link = rss.url + result.rss_link = rss.url + seen_titles.add(title_raw) + new_movies.append(result) + logger.info(f"New movie found: {result.official_title}") + elif isinstance(result, Bangumi): + await self.official_title_parser( + bangumi=result, rss=rss, torrent=torrent + ) + result.rss_link = rss.url if not full_parse: - return [bangumi] - seen_titles.add(bangumi.title_raw) - new_data.append(bangumi) - logger.info(f"New bangumi founded: {bangumi.official_title}") - return new_data + return [result], new_movies + seen_titles.add(title_raw) + new_bangumi.append(result) + logger.info(f"New bangumi found: {result.official_title}") + return new_bangumi, new_movies async def torrent_to_data( self, torrent: Torrent, rss: RSSItem, fetch_poster: bool = True - ) -> Bangumi | None: - bangumi = await TitleParser.raw_parser(raw=torrent.name) - if bangumi: - await self.official_title_parser( - bangumi=bangumi, rss=rss, torrent=torrent, fetch_poster=fetch_poster - ) - bangumi.rss_link = rss.url - return bangumi + ) -> Bangumi | Movie | None: + result = await TitleParser.raw_parser(raw=torrent.name) + if result: + if isinstance(result, Movie): + await self.official_title_parser_movie( + movie=result, + rss=rss, + torrent=torrent, + fetch_poster=fetch_poster, + ) + result.rss_link = rss.url + else: + await self.official_title_parser( + bangumi=result, + rss=rss, + torrent=torrent, + fetch_poster=fetch_poster, + ) + result.rss_link = rss.url + return result return None async def rss_to_data( self, rss: RSSItem, engine: RSSEngine, full_parse: bool = True ) -> list[Bangumi]: rss_torrents = await self.get_rss_torrents(rss.url, full_parse) - torrents_to_add = await engine.db.bangumi.match_list(rss_torrents, rss.url) + # Filter out already-known movies first + torrents_after_movies = await engine.db.movie.match_list(rss_torrents, rss.url) + # Then filter out already-known bangumi + torrents_to_add = await engine.db.bangumi.match_list( + torrents_after_movies, rss.url + ) if not torrents_to_add: logger.debug("No new title has been found.") return [] - # New List - new_data = await self.torrents_to_data(torrents_to_add, rss, full_parse) - if new_data: - # Add to database - await engine.db.bangumi.add_all(new_data) - return new_data - else: - return [] + # Parse remaining torrents + new_bangumi, new_movies = await self.torrents_to_data( + torrents_to_add, rss, full_parse + ) + for movie in new_movies: + await engine.db.movie.add(movie) + if new_bangumi: + await engine.db.bangumi.add_all(new_bangumi) + return new_bangumi + return [] - async def link_to_data(self, rss: RSSItem) -> Bangumi | ResponseModel: + async def link_to_data(self, rss: RSSItem) -> Bangumi | Movie | ResponseModel: torrents = await self.get_rss_torrents(rss.url, False) if not torrents: return ResponseModel( diff --git a/backend/src/module/rss/engine.py b/backend/src/module/rss/engine.py index 8750ef28..b3c15a0a 100644 --- a/backend/src/module/rss/engine.py +++ b/backend/src/module/rss/engine.py @@ -10,7 +10,7 @@ from module.conf import settings from module.database import Database from module.database.bangumi import _groups_are_similar, match_bangumi_in_list from module.downloader import AddResult, DownloadClient -from module.models import Bangumi, Episode, ResponseModel, RSSItem, Torrent +from module.models import Bangumi, Episode, Movie, ResponseModel, RSSItem, Torrent from module.network import RequestContent from module.notification.events import ( DownloadFailureEvent, @@ -383,6 +383,46 @@ class RSSEngine: await self.db.commit() return events + async def download_movie(self, movie: Movie): + if not movie.rss_link: + return ResponseModel( + status=False, + status_code=406, + msg_en=f"Download movie {movie.official_title} failed: no RSS link.", + msg_zh=f"下载剧场版 {movie.official_title} 失败：缺少 RSS 链接。", + ) + async with RequestContent() as req: + filter_pattern = movie.filter.replace(",", "|") if movie.filter else "" + torrents = await req.get_torrents(movie.rss_link, filter_pattern) + if torrents: + async with DownloadClient() as client: + result = await client.add_torrent( + torrents, movie # type: ignore[arg-type] + ) + if result is AddResult.FAILED: + return ResponseModel( + status=False, + status_code=502, + msg_en=f"Download movie {movie.official_title} failed.", + msg_zh=f"下载剧场版 {movie.official_title} 失败。", + ) + for torrent in torrents: + torrent.downloaded = True + await self.db.torrent.add_all(torrents) + return ResponseModel( + status=True, + status_code=200, + msg_en=f"Download movie {movie.official_title} successfully.", + msg_zh=f"下载剧场版 {movie.official_title} 成功。", + ) + else: + return ResponseModel( + status=False, + status_code=406, + msg_en=f"[Engine] Download movie {movie.official_title} failed.", + msg_zh=f"下载剧场版 {movie.official_title} 失败。", + ) + async def download_bangumi(self, bangumi: Bangumi): async with RequestContent() as req: torrents = await req.get_torrents( diff --git a/backend/src/test/test_migrations_module.py b/backend/src/test/test_migrations_module.py index eafb5ccf..c5283cfe 100644 --- a/backend/src/test/test_migrations_module.py +++ b/backend/src/test/test_migrations_module.py @@ -244,6 +244,60 @@ def _columns(engine, table: str) -> set[str]: return {c["name"] for c in inspect(engine).get_columns(table)} +def _run_through_version( + engine: Engine, + version: int, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Run the real migration table only through one historical version.""" + historical = tuple(item for item in MIGRATIONS if item.version <= version) + with monkeypatch.context() as patch: + patch.setattr(migration_module, "MIGRATIONS", historical) + patch.setattr(migration_module, "CURRENT_SCHEMA_VERSION", version) + run_migrations(engine) + + +def _make_v18_movie_engine(monkeypatch: pytest.MonkeyPatch) -> Engine: + """A database produced by the divergent tokenizer dev branch where v18=Movie.""" + engine = _make_v0_engine() + _run_through_version(engine, 17, monkeypatch) + with engine.begin() as conn: + conn.execute(text("""CREATE TABLE movie ( + id INTEGER PRIMARY KEY, + official_title VARCHAR NOT NULL, + title_raw VARCHAR, + year INTEGER, + group_name VARCHAR, + dpi VARCHAR, + source VARCHAR, + subtitle VARCHAR, + poster_link VARCHAR, + rss_link VARCHAR, + added BOOLEAN NOT NULL DEFAULT 0, + deleted BOOLEAN NOT NULL DEFAULT 0, + save_path VARCHAR, + rule_name VARCHAR, + filter VARCHAR NOT NULL DEFAULT '' + )""")) + conn.execute(text("CREATE INDEX ix_movie_title_raw ON movie(title_raw)")) + conn.execute(text("CREATE INDEX ix_movie_deleted ON movie(deleted)")) + conn.execute( + text( + "INSERT INTO movie " + "(id, official_title, title_raw, year, filter) VALUES " + "(9, 'Preserved Movie', 'Preserved Movie', 2026, '')" + ) + ) + conn.execute( + text( + "INSERT INTO user (id, username, password) VALUES " + "(7, 'movie_dev_user', 'hashed-password')" + ) + ) + set_schema_version(conn, 18) + return engine + + class TestMigrationTable: def test_every_migration_has_an_already_applied_guard(self): """No migration may rely on the old hardcoded if-ladder.""" @@ -277,6 +331,7 @@ class TestRunMigrations: assert {"enabled", "created_at", "updated_at"} <= _columns(engine, "user") assert "auth_session" in inspector.get_table_names() assert "api_token" in inspector.get_table_names() + assert "movie" in inspector.get_table_names() user_indexes = {ix["name"] for ix in inspector.get_indexes("user")} assert "ix_user_username" in user_indexes @@ -344,6 +399,59 @@ class TestRunMigrations: indexes = {ix["name"]: ix for ix in inspector.get_indexes("llmcredential")} assert indexes["ix_llmcredential_provider_id"]["unique"] + def test_upgrades_auth_beta_v20_without_losing_tokens(self, monkeypatch): + engine = _make_v19_auth_engine() + _run_through_version(engine, 20, monkeypatch) + with engine.connect() as conn: + before = conn.execute( + text("SELECT id, token_hash, scope, prefix FROM api_token ORDER BY id") + ).all() + + run_migrations(engine) + + inspector = inspect(engine) + assert "movie" in inspector.get_table_names() + assert {"enabled", "created_at", "updated_at"} <= _columns(engine, "user") + with engine.connect() as conn: + after = conn.execute( + text("SELECT id, token_hash, scope, prefix FROM api_token ORDER BY id") + ).all() + assert get_schema_version(conn) == CURRENT_SCHEMA_VERSION + assert after == before + + def test_repairs_divergent_movie_v18_database(self, monkeypatch): + engine = _make_v18_movie_engine(monkeypatch) + + run_migrations(engine) + + inspector = inspect(engine) + assert {"movie", "auth_session", "api_token"} <= set( + inspector.get_table_names() + ) + assert {"enabled", "created_at", "updated_at"} <= _columns(engine, "user") + user_indexes = {index["name"] for index in inspector.get_indexes("user")} + assert {"ix_user_username", "ix_user_enabled"} <= user_indexes + token_indexes = {index["name"] for index in inspector.get_indexes("api_token")} + assert "ix_api_token_token_hash_scope" in token_indexes + with engine.connect() as conn: + assert ( + conn.execute( + text("SELECT official_title FROM movie WHERE id = 9") + ).scalar_one() + == "Preserved Movie" + ) + user = conn.execute( + text( + "SELECT username, enabled, created_at, updated_at " + "FROM user WHERE id = 7" + ) + ).one() + assert user[0] == "movie_dev_user" + assert user[1] == 1 + assert user[2] is not None + assert user[3] is not None + assert get_schema_version(conn) == CURRENT_SCHEMA_VERSION + def test_v20_rebuilds_v19_api_tokens_without_losing_data(self): engine = _make_v19_auth_engine() # Production upgrade paths may call metadata.create_all before the @@ -378,7 +486,7 @@ class TestRunMigrations: .mappings() .all() ) - assert get_schema_version(conn) == 20 + assert get_schema_version(conn) == CURRENT_SCHEMA_VERSION assert len(after) == len(before) == 2 for old, new in zip(before, after): @@ -420,7 +528,7 @@ class TestRunMigrations: def test_v20_rebuild_rolls_back_if_a_late_statement_fails(self, monkeypatch): engine = _make_v19_auth_engine() - v20 = MIGRATIONS[-1] + v20 = next(item for item in MIGRATIONS if item.version == 20) broken_v20 = replace( v20, statements=( @@ -432,7 +540,7 @@ class TestRunMigrations: monkeypatch.setattr( migration_module, "MIGRATIONS", - (*MIGRATIONS[:-1], broken_v20), + tuple(broken_v20 if item.version == 20 else item for item in MIGRATIONS), ) with pytest.raises(OperationalError): @@ -481,7 +589,7 @@ class TestRunMigrations: prefix = conn.execute( text("SELECT prefix FROM api_token WHERE id = 1") ).scalar_one() - assert get_schema_version(conn) == 20 + assert get_schema_version(conn) == CURRENT_SCHEMA_VERSION assert prefix == "legacy_cccccccc" def test_is_idempotent(self): diff --git a/backend/src/test/test_movie_parser.py b/backend/src/test/test_movie_parser.py new file mode 100644 index 00000000..3315353b --- /dev/null +++ b/backend/src/test/test_movie_parser.py @@ -0,0 +1,81 @@ +import pytest + +from module.parser.analyser import raw_parser + + +class TestMovieMarkerDetection: + def test_gekijouban_chinese_traditional(self): + content = "[SubGroup] 劇場版 Some Anime Title [1080p]" + info = raw_parser(content) + assert info is not None + assert info.is_movie is True + + def test_gekijouban_chinese_simplified(self): + content = "[SubGroup] 剧场版 Some Anime Title [1080p]" + info = raw_parser(content) + assert info is not None + assert info.is_movie is True + + def test_movie_keyword_english(self): + content = "[SubGroup] Some Anime Title Movie [1080p HEVC]" + info = raw_parser(content) + assert info is not None + assert info.is_movie is True + + def test_gekijouban_romaji(self): + content = "[SubGroup] Gekijouban Some Anime [1080p]" + info = raw_parser(content) + assert info is not None + assert info.is_movie is True + + def test_movie_keyword_case_insensitive(self): + content = "[SubGroup] Some Anime MOVIE [720p]" + info = raw_parser(content) + assert info is not None + assert info.is_movie is True + + def test_gekijouban_with_hyphen(self): + content = "[SubGroup] Gekijou-ban Title [1080p]" + info = raw_parser(content) + assert info is not None + assert info.is_movie is True + + def test_movie_defaults(self): + """Movies should default to episode=0, season=1.""" + content = "[SubGroup] 劇場版 Anime Title [1080p]" + info = raw_parser(content) + assert info is not None + assert info.is_movie is True + assert info.episode == 0 + assert info.season == 1 + + +class TestNonMovieRegression: + def test_regular_episode_not_movie(self): + content = "[LoliHouse] Some Anime - 01 [WebRip 1080p HEVC-10bit AAC]" + info = raw_parser(content) + assert info is not None + assert info.is_movie is False + assert info.episode == 1 + + def test_regular_series_with_season(self): + content = "【幻樱字幕组】【Anime Title S02】【12】【GB_MP4】【1920X1080】" + info = raw_parser(content) + assert info is not None + assert info.is_movie is False + assert info.season == 2 + assert info.episode == 12 + + def test_regular_series_chinese_episode(self): + content = "[字幕组] 动画标题 第5话 [1080p]" + info = raw_parser(content) + assert info is not None + assert info.is_movie is False + assert info.episode == 5 + + def test_end_marker_not_movie(self): + content = "[LoliHouse] Anime Title - 12 [WebRip 1080p HEVC-10bit AAC][END]" + info = raw_parser(content) + assert info is not None + assert info.is_movie is False + assert info.episode == 12