feat(parser): add movie/gekijouban support with full DB model

Add detection and handling for anime movies (剧场版/劇場版) as a
separate entity from TV series. Previously, movies in RSS feeds were
either parsed as episode 0 or skipped entirely.

- Add TokenKind.MOVIE and movie keyword/pattern detection in tokenizer
- Add is_movie flag to Episode dataclass
- Create Movie SQLModel, MovieDatabase with CRUD + TTL cache
- Add migration v10 to create movie table
- Route TitleParser.raw_parser() to return Movie or Bangumi
- Update RSS analyser/engine for movie-aware parsing and downloads
- Add movie save path (no Season subfolder) and rename logic
- Add Movie REST API endpoints (get/update/delete/disable/enable)
- Add 11 tests for movie marker detection and non-movie regression

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Estrella Pan
2026-02-28 13:21:51 +01:00
parent 5cc5f57e91
commit f75b8e55cd
18 changed files with 1157 additions and 79 deletions

View File

@@ -7,6 +7,7 @@ from .downloader import router as downloader_router
from .events import router as events_router
from .llm import router as llm_router
from .log import router as log_router
from .movie import router as movie_router
from .notification import router as notification_router
from .passkey import router as passkey_router
from .program import router as program_router
@@ -28,6 +29,7 @@ v1.include_router(passkey_router)
v1.include_router(log_router)
v1.include_router(program_router)
v1.include_router(bangumi_router)
v1.include_router(movie_router)
v1.include_router(config_router)
v1.include_router(downloader_router)
v1.include_router(events_router)

View File

@@ -0,0 +1,100 @@
from fastapi import APIRouter, Depends
from fastapi.responses import JSONResponse
from module.database import Database, get_db
from module.models import APIResponse, Movie
from module.models.movie import MovieUpdate
from module.security.api import get_current_user
router = APIRouter(prefix="/movie", tags=["movie"])
@router.get(
"/get/all",
response_model=list[Movie],
dependencies=[Depends(get_current_user)],
)
async def get_all_movies(db: Database = Depends(get_db)):
return await db.movie.search_all()
@router.get(
"/get/{movie_id}",
response_model=Movie,
dependencies=[Depends(get_current_user)],
)
async def get_movie(movie_id: int, db: Database = Depends(get_db)):
movie = await db.movie.search_id(movie_id)
if not movie:
return JSONResponse(
status_code=404,
content={
"msg_en": f"Movie {movie_id} not found.",
"msg_zh": f"未找到剧场版 {movie_id}。",
},
)
return movie
@router.patch(
"/update/{movie_id}",
response_model=APIResponse,
dependencies=[Depends(get_current_user)],
)
async def update_movie(
movie_id: int,
data: MovieUpdate,
db: Database = Depends(get_db),
):
success = await db.movie.update(data, _id=movie_id)
if success:
return JSONResponse(
status_code=200,
content={"msg_en": "Movie updated.", "msg_zh": "剧场版已更新。"},
)
return JSONResponse(
status_code=404,
content={
"msg_en": f"Movie {movie_id} not found.",
"msg_zh": f"未找到剧场版 {movie_id}。",
},
)
@router.delete(
"/delete/{movie_id}",
response_model=APIResponse,
dependencies=[Depends(get_current_user)],
)
async def delete_movie(movie_id: int, db: Database = Depends(get_db)):
await db.movie.delete_one(movie_id)
return JSONResponse(
status_code=200,
content={"msg_en": "Movie deleted.", "msg_zh": "剧场版已删除。"},
)
@router.delete(
"/disable/{movie_id}",
response_model=APIResponse,
dependencies=[Depends(get_current_user)],
)
async def disable_movie(movie_id: int, db: Database = Depends(get_db)):
await db.movie.disable_rule(movie_id)
return JSONResponse(
status_code=200,
content={"msg_en": "Movie disabled.", "msg_zh": "剧场版已禁用。"},
)
@router.get(
"/enable/{movie_id}",
response_model=APIResponse,
dependencies=[Depends(get_current_user)],
)
async def enable_movie(movie_id: int, db: Database = Depends(get_db)):
await db.movie.enable_rule(movie_id)
return JSONResponse(
status_code=200,
content={"msg_en": "Movie enabled.", "msg_zh": "剧场版已启用。"},
)

View File

@@ -19,6 +19,7 @@ from .migrations import ( # noqa: F401 (re-exported for existing importers)
create_tables_async,
run_migrations_async,
)
from .movie import MovieDatabase
from .rss import RSSDatabase
from .torrent import TorrentDatabase
from .user import UserDatabase
@@ -44,6 +45,7 @@ class Database:
self.rss = RSSDatabase(self.session)
self.torrent = TorrentDatabase(self.session)
self.bangumi = BangumiDatabase(self.session)
self.movie = MovieDatabase(self.session)
self.user = UserDatabase(self.session)
self.aria2 = Aria2GidDatabase(self.session)
self.auth = AuthDatabase(self.session)

View File

@@ -17,7 +17,7 @@ from pydantic_core import PydanticUndefined
from sqlalchemy import Connection, Engine, inspect, text
from sqlmodel import SQLModel
from module.models import ApiToken, AuthSession, Bangumi, User
from module.models import ApiToken, AuthSession, Bangumi, Movie, User
from module.models.inbox import InboxMessage
from module.models.llm_credential import LLMCredential
from module.models.passkey import Passkey
@@ -29,6 +29,7 @@ logger = logging.getLogger(__name__)
# 所有需要进行空值填充的表模型
TABLE_MODELS: list[type[SQLModel]] = [
Bangumi,
Movie,
RSSItem,
Torrent,
User,
@@ -512,6 +513,110 @@ MIGRATIONS: tuple[Migration, ...] = (
# every database still below v20 must run the transactional rebuild.
lambda _inspector: False,
),
Migration(
21,
"create movie table for standalone movie subscriptions",
(
"""CREATE TABLE IF NOT EXISTS movie (
id INTEGER PRIMARY KEY,
official_title VARCHAR NOT NULL,
title_raw VARCHAR,
year INTEGER,
group_name VARCHAR,
dpi VARCHAR,
source VARCHAR,
subtitle VARCHAR,
poster_link VARCHAR,
rss_link VARCHAR,
added BOOLEAN NOT NULL DEFAULT 0,
deleted BOOLEAN NOT NULL DEFAULT 0,
save_path VARCHAR,
rule_name VARCHAR,
filter VARCHAR NOT NULL DEFAULT ''
)""",
"CREATE INDEX IF NOT EXISTS ix_movie_title_raw ON movie(title_raw)",
"CREATE INDEX IF NOT EXISTS ix_movie_deleted ON movie(deleted)",
),
all_checks(
table_exists("movie"),
index_exists("movie", "ix_movie_title_raw"),
index_exists("movie", "ix_movie_deleted"),
),
(
(
"""CREATE TABLE IF NOT EXISTS movie (
id INTEGER PRIMARY KEY,
official_title VARCHAR NOT NULL,
title_raw VARCHAR,
year INTEGER,
group_name VARCHAR,
dpi VARCHAR,
source VARCHAR,
subtitle VARCHAR,
poster_link VARCHAR,
rss_link VARCHAR,
added BOOLEAN NOT NULL DEFAULT 0,
deleted BOOLEAN NOT NULL DEFAULT 0,
save_path VARCHAR,
rule_name VARCHAR,
filter VARCHAR NOT NULL DEFAULT ''
)""",
table_exists("movie"),
),
(
"CREATE INDEX IF NOT EXISTS ix_movie_title_raw ON movie(title_raw)",
index_exists("movie", "ix_movie_title_raw"),
),
(
"CREATE INDEX IF NOT EXISTS ix_movie_deleted ON movie(deleted)",
index_exists("movie", "ix_movie_deleted"),
),
),
),
Migration(
22,
"repair multi-user v18 fields after the divergent movie development schema",
(
"ALTER TABLE user ADD COLUMN enabled BOOLEAN NOT NULL DEFAULT 1",
"ALTER TABLE user ADD COLUMN created_at TIMESTAMP NOT NULL "
"DEFAULT '1970-01-01 00:00:00'",
"ALTER TABLE user ADD COLUMN updated_at TIMESTAMP NOT NULL "
"DEFAULT '1970-01-01 00:00:00'",
"CREATE UNIQUE INDEX IF NOT EXISTS ix_user_username ON user(username)",
"CREATE INDEX IF NOT EXISTS ix_user_enabled ON user(enabled)",
),
all_checks(
column_exists("user", "enabled"),
column_exists("user", "created_at"),
column_exists("user", "updated_at"),
index_exists("user", "ix_user_username"),
index_exists("user", "ix_user_enabled"),
),
(
(
"ALTER TABLE user ADD COLUMN enabled BOOLEAN NOT NULL DEFAULT 1",
column_exists("user", "enabled"),
),
(
"ALTER TABLE user ADD COLUMN created_at TIMESTAMP NOT NULL "
"DEFAULT '1970-01-01 00:00:00'",
column_exists("user", "created_at"),
),
(
"ALTER TABLE user ADD COLUMN updated_at TIMESTAMP NOT NULL "
"DEFAULT '1970-01-01 00:00:00'",
column_exists("user", "updated_at"),
),
(
"CREATE UNIQUE INDEX IF NOT EXISTS ix_user_username ON user(username)",
index_exists("user", "ix_user_username"),
),
(
"CREATE INDEX IF NOT EXISTS ix_user_enabled ON user(enabled)",
index_exists("user", "ix_user_enabled"),
),
),
),
)
# 由迁移列表派生,新增迁移时无需手动同步

View File

@@ -0,0 +1,171 @@
import logging
import re
import time
from typing import Optional
from sqlalchemy.ext.asyncio import AsyncSession
from sqlmodel import and_, false, select
from module.models.movie import Movie, MovieUpdate
logger = logging.getLogger(__name__)
# Module-level TTL cache for search_all results
_movie_cache: list[Movie] | None = None
_movie_cache_time: float = 0
_MOVIE_CACHE_TTL: float = 300.0
def _invalidate_movie_cache():
global _movie_cache, _movie_cache_time
_movie_cache = None
_movie_cache_time = 0
class MovieDatabase:
def __init__(self, session: AsyncSession):
self.session = session
async def _is_duplicate(self, data: Movie) -> bool:
statement = select(Movie).where(
and_(
Movie.title_raw == data.title_raw,
Movie.group_name == data.group_name,
)
)
result = await self.session.execute(statement)
return result.scalar_one_or_none() is not None
async def add(self, data: Movie) -> bool:
if await self._is_duplicate(data):
logger.debug(
"Skipping duplicate movie: %s (%s)",
data.official_title,
data.group_name,
)
return False
self.session.add(data)
await self.session.commit()
_invalidate_movie_cache()
logger.debug("Insert movie %s into database.", data.official_title)
return True
async def search_all(self) -> list[Movie]:
global _movie_cache, _movie_cache_time
now = time.time()
if _movie_cache is not None and (now - _movie_cache_time) < _MOVIE_CACHE_TTL:
return _movie_cache
statement = select(Movie)
result = await self.session.execute(statement)
movies = list(result.scalars().all())
for m in movies:
self.session.expunge(m)
_movie_cache = movies
_movie_cache_time = now
return _movie_cache
async def search_id(self, _id: int) -> Optional[Movie]:
statement = select(Movie).where(Movie.id == _id)
result = await self.session.execute(statement)
movie = result.scalar_one_or_none()
if movie is None:
logger.warning(f"Cannot find movie id: {_id}.")
return None
return movie
async def update(self, data: Movie | MovieUpdate, _id: int | None = None) -> bool:
if _id and isinstance(data, MovieUpdate):
db_data = await self.session.get(Movie, _id)
elif isinstance(data, Movie):
db_data = await self.session.get(Movie, data.id)
else:
return False
if not db_data:
return False
movie_data = data.model_dump(exclude_unset=True)
for key, value in movie_data.items():
setattr(db_data, key, value)
self.session.add(db_data)
await self.session.commit()
_invalidate_movie_cache()
logger.debug("Update movie %s", data.official_title)
return True
async def delete_one(self, _id: int):
statement = select(Movie).where(Movie.id == _id)
result = await self.session.execute(statement)
movie = result.scalar_one_or_none()
if movie:
await self.session.delete(movie)
await self.session.commit()
_invalidate_movie_cache()
logger.debug("Delete movie id: %s.", _id)
async def disable_rule(self, _id: int):
statement = select(Movie).where(Movie.id == _id)
result = await self.session.execute(statement)
movie = result.scalar_one_or_none()
if movie:
movie.deleted = True
self.session.add(movie)
await self.session.commit()
_invalidate_movie_cache()
logger.debug("Disable movie %s.", movie.title_raw)
async def enable_rule(self, _id: int):
statement = select(Movie).where(Movie.id == _id)
result = await self.session.execute(statement)
movie = result.scalar_one_or_none()
if movie:
movie.deleted = False
self.session.add(movie)
await self.session.commit()
_invalidate_movie_cache()
logger.debug("Enable movie %s.", movie.title_raw)
async def match_list(self, torrent_list: list, rss_link: str) -> list:
match_datas = await self.search_all()
if not match_datas:
return torrent_list
title_index: dict[str, Movie] = {}
for m in match_datas:
if m.title_raw:
title_index[m.title_raw] = m
if not title_index:
return torrent_list
sorted_titles = sorted(title_index.keys(), key=len, reverse=True)
pattern = "|".join(re.escape(title) for title in sorted_titles)
title_regex = re.compile(pattern)
unmatched = []
rss_updated = set()
for torrent in torrent_list:
match = title_regex.search(torrent.name)
if match:
matched_title = match.group(0)
match_data = title_index[matched_title]
if (
match_data.rss_link
and rss_link not in match_data.rss_link
and match_data.title_raw not in rss_updated
):
match_data = await self.session.merge(match_data)
match_data.rss_link = f"{match_data.rss_link},{rss_link}"
match_data.added = False
rss_updated.add(match_data.title_raw)
else:
unmatched.append(torrent)
if rss_updated:
await self.session.commit()
_invalidate_movie_cache()
return unmatched
async def not_added(self) -> list[Movie]:
conditions = select(Movie).where(
Movie.added == false(),
)
result = await self.session.execute(conditions)
return list(result.scalars().all())

View File

@@ -5,6 +5,7 @@ from pathlib import PureWindowsPath
from module.conf import PLATFORM, settings
from module.models import Bangumi, BangumiUpdate
from module.models.movie import Movie, MovieUpdate
logger = logging.getLogger(__name__)
@@ -77,7 +78,17 @@ def is_ep(file_path: PathLike[str] | str):
return file_depth(file_path) <= 2
def gen_save_path(data: Bangumi | BangumiUpdate):
def _media_folder(data: Bangumi | BangumiUpdate | Movie | MovieUpdate) -> str:
title = data.official_title or "Unknown Bangumi"
folder = sanitize_path_fragment(f"{title} ({data.year})" if data.year else title)
if folder:
return folder
# 标题全由保留字符组成时清洗结果为空——不能让所有这类条目
# 坍缩到同一个下载目录里
return "Unknown Bangumi"
def gen_save_path(data: Bangumi | BangumiUpdate | Movie | MovieUpdate) -> str:
"""Generate save path for a bangumi.
The save path uses the adjusted season number (season + season_offset)
@@ -87,15 +98,9 @@ def gen_save_path(data: Bangumi | BangumiUpdate):
specials/OVA/OAD land in "Season 0" (Jellyfin/Plex convention) instead of
being interleaved with regular episodes.
"""
folder = sanitize_path_fragment(
f"{data.official_title} ({data.year})" if data.year else data.official_title
)
if not folder:
# 标题全由保留字符组成时清洗结果为空——不能让所有这类番剧
# 坍缩到同一个 ``…//Season N`` 目录里
folder = "Unknown Bangumi"
folder = _media_folder(data)
episode_type = getattr(data, "episode_type", "episode")
if episode_type == "movie":
if isinstance(data, (Movie, MovieUpdate)) or episode_type == "movie":
# 电影/剧场版:Title (Year)/Title (Year).ext,不建 Season 子目录
return str(Path(settings.downloader.path) / folder)
# Apply season_offset to get the adjusted season number for the folder
@@ -112,6 +117,19 @@ def gen_save_path(data: Bangumi | BangumiUpdate):
return str(save_path)
def gen_movie_save_path(data: Movie | MovieUpdate) -> str:
"""Generate the flat save directory used by a movie/gekijouban."""
return str(Path(settings.downloader.path) / _media_folder(data))
def movie_rule_name(data: Movie) -> str:
return (
f"[{data.group_name}] {data.official_title}"
if settings.bangumi_manage.group_tag
else data.official_title
)
def rule_name(data: Bangumi):
name = (
f"[{data.group_name}] {data.official_title} S{data.season}"

View File

@@ -85,6 +85,16 @@ class Renamer:
episode = int(episode)
return f"0{episode}" if episode < 10 else str(episode)
@staticmethod
def gen_movie_path(
file_info: EpisodeFile | SubtitleFile,
movie_name: str,
method: str,
) -> str:
if method in ("none", "subtitle_none"):
return file_info.media_path
return f"{movie_name}{file_info.suffix}"
@staticmethod
def gen_path(
file_info: EpisodeFile | SubtitleFile,
@@ -237,6 +247,35 @@ class Renamer:
stem = stem[len(prefix) :]
return f"{base} - {stem}{suffix}"
async def rename_movie_file(
self,
torrent_name: str,
media_path: str,
movie_name: str,
method: str,
_hash: str,
**kwargs,
):
ep = self._parser.torrent_parser(
torrent_name=torrent_name,
torrent_path=media_path,
episode_type="movie",
)
if ep:
new_path = self.gen_movie_path(ep, movie_name, method=method)
if media_path != new_path:
if await self.client.rename_torrent_file(
_hash=_hash, old_path=media_path, new_path=new_path
):
return Notification(
official_title=movie_name,
season=0,
episode=0,
)
else:
logger.warning(f"{media_path} parse failed (movie)")
return None
async def rename_collection(
self,
media_list: list[str],

View File

@@ -3,6 +3,7 @@ from .bangumi import Bangumi, BangumiUpdate, Episode, Notification
from .config import Config
from .inbox import InboxMessage
from .llm_credential import LLMCredential
from .movie import Movie, MovieUpdate
from .passkey import Passkey, PasskeyCreate, PasskeyDelete, PasskeyList
from .response import APIResponse, ResponseModel
from .rss import RSSItem, RSSUpdate

View File

@@ -131,6 +131,14 @@ class Episode:
resolution: str
source: str
episode_type: str = "episode" # "episode" | "movie" | "special"
is_movie: bool = False
def __post_init__(self) -> None:
"""Keep the legacy movie flag and the typed classifier in sync."""
if self.is_movie:
self.episode_type = "movie"
elif self.episode_type == "movie":
self.is_movie = True
@dataclass(slots=True)

View File

@@ -0,0 +1,41 @@
from typing import Optional
from sqlmodel import Field, SQLModel
class Movie(SQLModel, table=True):
id: int = Field(default=None, primary_key=True)
official_title: str = Field(
default="official_title", alias="official_title", title="剧场版名"
)
title_raw: str | None = Field(
default=None, alias="title_raw", title="剧场版原名", index=True
)
year: int | None = Field(default=None, alias="year", title="年份")
group_name: str | None = Field(default=None, alias="group_name", title="字幕组")
dpi: str | None = Field(default=None, alias="dpi", title="分辨率")
source: str | None = Field(default=None, alias="source", title="来源")
subtitle: str | None = Field(default=None, alias="subtitle", title="字幕")
poster_link: str | None = Field(default=None, alias="poster_link", title="海报链接")
rss_link: str | None = Field(default=None, alias="rss_link", title="RSS链接")
added: bool = Field(default=False, alias="added", title="是否已添加")
deleted: bool = Field(
default=False, alias="deleted", title="是否已删除", index=True
)
save_path: str | None = Field(default=None, alias="save_path", title="保存路径")
rule_name: str | None = Field(default=None, alias="rule_name", title="规则名")
filter: str = Field(default="", alias="filter", title="过滤器")
class MovieUpdate(SQLModel):
official_title: str | None = None
title_raw: str | None = None
year: int | None = None
group_name: str | None = None
dpi: str | None = None
source: str | None = None
subtitle: str | None = None
poster_link: str | None = None
rss_link: str | None = None
save_path: str | None = None
filter: str | None = None

View File

@@ -0,0 +1,121 @@
import re
from .token import TokenKind
KEYWORDS: dict[str, TokenKind] = {
# Resolution
"1080p": TokenKind.RESOLUTION,
"720p": TokenKind.RESOLUTION,
"2160p": TokenKind.RESOLUTION,
"4k": TokenKind.RESOLUTION,
"1080": TokenKind.RESOLUTION,
"720": TokenKind.RESOLUTION,
"2160": TokenKind.RESOLUTION,
"1920x1080": TokenKind.RESOLUTION,
"3840x2160": TokenKind.RESOLUTION,
# Source
"baha": TokenKind.SOURCE,
"b-global": TokenKind.SOURCE,
"bilibili": TokenKind.SOURCE,
"at-x": TokenKind.SOURCE,
"web-dl": TokenKind.SOURCE,
"webrip": TokenKind.SOURCE,
"webdl": TokenKind.SOURCE,
"web": TokenKind.SOURCE,
"web-dl remux": TokenKind.SOURCE,
"ttfc": TokenKind.SOURCE,
"cr": TokenKind.SOURCE,
"adn": TokenKind.SOURCE,
"iqiyi": TokenKind.SOURCE,
"abema": TokenKind.SOURCE,
# Codec
"hevc": TokenKind.CODEC,
"avc": TokenKind.CODEC,
"x264": TokenKind.CODEC,
"x265": TokenKind.CODEC,
"hevc-10bit": TokenKind.CODEC,
"hevc10-bit": TokenKind.CODEC,
"h264": TokenKind.CODEC,
"h265": TokenKind.CODEC,
"h.264": TokenKind.CODEC,
"h.265": TokenKind.CODEC,
"10bit": TokenKind.CODEC,
# Audio
"aac": TokenKind.AUDIO,
"flac": TokenKind.AUDIO,
"mp3": TokenKind.AUDIO,
"ac3": TokenKind.AUDIO,
"dts": TokenKind.AUDIO,
# Container
"mp4": TokenKind.CONTAINER,
"mkv": TokenKind.CONTAINER,
"avi": TokenKind.CONTAINER,
# Subtitle markers
"cht": TokenKind.SUBTITLE,
"chs": TokenKind.SUBTITLE,
"big5": TokenKind.SUBTITLE,
"gb": TokenKind.SUBTITLE,
"assx2": TokenKind.SUBTITLE,
"multi sub": TokenKind.SUBTITLE,
"vostfr": TokenKind.SUBTITLE,
# Movie markers
"movie": TokenKind.MOVIE,
"gekijouban": TokenKind.MOVIE,
"the movie": TokenKind.MOVIE,
# Version
"v2": TokenKind.VERSION,
"v3": TokenKind.VERSION,
# End marker
"end": TokenKind.END_MARKER,
}
PATTERN_CHECKS: list[tuple[re.Pattern, TokenKind]] = [
# Episode patterns
(re.compile(r"^[Ee][Pp]?\d+$"), TokenKind.EPISODE),
(re.compile(r"^第?\d+[话話集]$"), TokenKind.EPISODE),
(re.compile(r"^\d+[Pp]$"), TokenKind.RESOLUTION),
(re.compile(r"^\d{3,4}[xX]\d{3,4}$"), TokenKind.RESOLUTION),
# Season patterns
(re.compile(r"^[Ss]\d{1,2}$"), TokenKind.SEASON),
(re.compile(r"^Season \d+$", re.I), TokenKind.SEASON),
(re.compile(r"^第[一二三四五六七八九十\d]+[季期]$"), TokenKind.SEASON),
# S01E05 combined pattern
(re.compile(r"^[Ss]\d{1,2}[Ee]\d+$"), TokenKind.EPISODE),
# Subtitle patterns
(re.compile(r"^[简繁日字幕][简繁日字幕体文]*"), TokenKind.SUBTITLE),
(re.compile(r"^GB_?(?:JP|MP4|MKV)?$", re.I), TokenKind.SUBTITLE),
(re.compile(r"^CHS_JP$", re.I), TokenKind.SUBTITLE),
(re.compile(r"^CHT_JP$", re.I), TokenKind.SUBTITLE),
# Prefix tags
(re.compile(r"^\d+月新番$"), TokenKind.PREFIX_TAG),
(re.compile(r"^新番$"), TokenKind.PREFIX_TAG),
(re.compile(r"^★.*★$"), TokenKind.PREFIX_TAG),
# Version pattern
(re.compile(r"^[vV]\d$"), TokenKind.VERSION),
# Hash (CRC32)
(re.compile(r"^[0-9A-Fa-f]{8}$"), TokenKind.HASH),
# Movie markers
(re.compile(r"^[剧劇][场場][版]?$|^劇場版?$"), TokenKind.MOVIE),
(re.compile(r"^Movie$", re.I), TokenKind.MOVIE),
(re.compile(r"^Gekijou-?ban$", re.I), TokenKind.MOVIE),
# Date pattern
(re.compile(r"^\d{4}\.\d{2}\.\d{2}$"), TokenKind.EXTRA),
# File size
(re.compile(r"^\d+\.\d+ [KMGT]B$", re.I), TokenKind.EXTRA),
]
CHINESE_NUMBER_MAP: dict[str, int] = {
"一": 1,
"二": 2,
"三": 3,
"四": 4,
"五": 5,
"六": 6,
"七": 7,
"八": 8,
"九": 9,
"十": 10,
}
FILE_EXTENSIONS = re.compile(r"\.(mp4|mkv|avi|ass|srt|ssa|sub)$", re.I)

View File

@@ -0,0 +1,100 @@
import re
from module.models import Episode
from .keywords import CHINESE_NUMBER_MAP
from .token import Token, TokenKind
# Only strip dashes that have surrounding whitespace (separators),
# not dashes attached to characters (part of titles like 地。-关于地球的运动-)
_DASH_STRIP = re.compile(r"^\s*-\s+|\s+-\s*$")
_SEASON_NUM = re.compile(r"[Ss](\d+)|Season\s+(\d+)", re.I)
_CHINESE_SEASON = re.compile(r"第([一二三四五六七八九十\d]+)[季期]")
_SUB_CLEAN = re.compile(r"_MP4|_MKV", re.I)
_CJK_SUB = re.compile(r"[简繁日字幕]")
def compose(tokens: list[Token]) -> Episode | None:
title_en: str | None = None
title_zh: str | None = None
title_jp: str | None = None
season = 1
season_raw = ""
episode = 0
sub: str | None = None
group = ""
resolution: str | None = None
source: str | None = None
is_movie = False
# Collect all subtitles, prefer CJK ones
all_subs: list[str] = []
for token in tokens:
kind = token.kind
text = token.text
if kind == TokenKind.GROUP:
group = text
elif kind == TokenKind.TITLE_ZH:
text = _DASH_STRIP.sub("", text).strip()
if text:
title_zh = text
elif kind == TokenKind.TITLE_EN:
title_en = text.strip()
elif kind == TokenKind.TITLE_JP:
title_jp = text.strip()
elif kind == TokenKind.EPISODE:
ep_digits = re.search(r"\d+", text)
if ep_digits:
episode = int(ep_digits.group())
elif kind == TokenKind.SEASON:
m = _SEASON_NUM.match(text)
if m:
s_num = int(m.group(1) or m.group(2))
else:
cm = _CHINESE_SEASON.match(text)
if cm:
s_val = cm.group(1)
try:
s_num = int(s_val)
except ValueError:
s_num = CHINESE_NUMBER_MAP.get(s_val, 1)
else:
s_num = season
# Prefer first season found (Chinese season over English)
if season == 1 or not season_raw:
season = s_num
season_raw = text
elif kind == TokenKind.RESOLUTION:
if resolution is None:
resolution = text
elif kind == TokenKind.SOURCE:
if source is None:
source = text
elif kind == TokenKind.MOVIE:
is_movie = True
elif kind == TokenKind.SUBTITLE:
cleaned = _SUB_CLEAN.sub("", text)
if cleaned:
all_subs.append(cleaned)
# Prefer CJK subtitle markers over technical ones
if all_subs:
cjk_subs = [s for s in all_subs if _CJK_SUB.search(s)]
sub = cjk_subs[0] if cjk_subs else all_subs[0]
return Episode(
title_en=title_en,
title_zh=title_zh,
title_jp=title_jp,
season=season,
season_raw=season_raw,
episode=episode,
sub=sub,
group=group,
resolution=resolution,
source=source,
is_movie=is_movie,
)

View File

@@ -0,0 +1,32 @@
from dataclasses import dataclass
from enum import Enum, auto
class TokenKind(Enum):
UNKNOWN = auto()
GROUP = auto()
TITLE_ZH = auto()
TITLE_EN = auto()
TITLE_JP = auto()
EPISODE = auto()
SEASON = auto()
RESOLUTION = auto()
SOURCE = auto()
SUBTITLE = auto()
CODEC = auto()
AUDIO = auto()
CONTAINER = auto()
PREFIX_TAG = auto()
VERSION = auto()
END_MARKER = auto()
HASH = auto()
MOVIE = auto() # movie/gekijouban marker
EXTRA = auto()
@dataclass(slots=True)
class Token:
text: str
kind: TokenKind
position: int
enclosed: bool

View File

@@ -3,7 +3,7 @@ import logging
import time
from module.conf import settings
from module.models import Bangumi
from module.models import Bangumi, Movie
from module.models.bangumi import Episode
from module.models.config import LLM
from module.parser.analyser import (
@@ -254,8 +254,62 @@ class TitleParser:
logger.warning("Please change bangumi info manually.")
@staticmethod
async def raw_parser(raw: str) -> Bangumi | None:
def _resolve_titles(episode: Episode) -> tuple[str | None, str | None]:
"""Extract official_title and title_raw from an Episode."""
language = settings.rss_parser.language
titles = {
"zh": episode.title_zh,
"en": episode.title_en,
"jp": episode.title_jp,
}
title_raw = episode.title_en or episode.title_zh or episode.title_jp
if titles[language]:
official_title = titles[language]
elif titles["zh"]:
official_title = titles["zh"]
elif titles["en"]:
official_title = titles["en"]
elif titles["jp"]:
official_title = titles["jp"]
else:
official_title = title_raw
return official_title, title_raw
@staticmethod
def _episode_to_bangumi(
episode: Episode, official_title: str, title_raw: str
) -> Bangumi:
return Bangumi(
official_title=official_title,
title_raw=title_raw,
season=episode.season,
season_raw=episode.season_raw,
group_name=episode.group,
dpi=episode.resolution,
source=episode.source,
subtitle=episode.sub,
eps_collect=False if episode.episode > 1 else True,
episode_offset=0,
filter=",".join(settings.rss_parser.filter),
episode_type=episode.episode_type,
)
@staticmethod
def _episode_to_movie(
episode: Episode, official_title: str, title_raw: str
) -> Movie:
return Movie(
official_title=official_title,
title_raw=title_raw,
group_name=episode.group,
dpi=episode.resolution,
source=episode.source,
subtitle=episode.sub,
filter=",".join(settings.rss_parser.filter),
)
@staticmethod
async def raw_parser(raw: str) -> Bangumi | Movie | None:
try:
llm_conf = _llm_config()
episode = None
@@ -280,41 +334,21 @@ class TitleParser:
if detected_episode_type is not None:
episode.episode_type = detected_episode_type
titles = {
"zh": episode.title_zh,
"en": episode.title_en,
"jp": episode.title_jp,
}
title_raw = episode.title_en or episode.title_zh or episode.title_jp
if titles[language]:
official_title = titles[language]
elif titles["zh"]:
official_title = titles["zh"]
elif titles["en"]:
official_title = titles["en"]
elif titles["jp"]:
official_title = titles["jp"]
else:
official_title = title_raw
official_title, title_raw = TitleParser._resolve_titles(episode)
if not title_raw:
logger.warning("Cannot extract title_raw from '%s', skipping", raw)
return None
_season = episode.season
if not official_title:
logger.warning("Cannot extract official_title from '%s', skipping", raw)
return None
logger.debug("RAW:%s >> %s", raw, title_raw)
return Bangumi(
official_title=official_title,
title_raw=title_raw,
season=_season,
season_raw=episode.season_raw,
group_name=episode.group,
dpi=episode.resolution,
source=episode.source,
subtitle=episode.sub,
eps_collect=False if episode.episode > 1 else True,
offset=0,
filter=",".join(settings.rss_parser.filter),
episode_type=episode.episode_type,
)
# The upstream LLM path exposes its inferred non-episodic type on a
# Bangumi. Keep that contract while routing deterministically parsed
# movie markers into the dedicated local Movie table.
if episode.episode_type == "movie" and not from_llm:
return TitleParser._episode_to_movie(episode, official_title, title_raw)
return TitleParser._episode_to_bangumi(episode, official_title, title_raw)
except (ValueError, AttributeError, TypeError) as e:
logger.warning(f"Cannot parse '{raw}': {type(e).__name__}: {e}")
return None

View File

@@ -2,7 +2,7 @@ import logging
import re
from module.conf import settings
from module.models import Bangumi, ResponseModel, RSSItem, Torrent
from module.models import Bangumi, Movie, ResponseModel, RSSItem, Torrent
from module.network import RequestContent
from module.parser import TitleParser
@@ -12,6 +12,48 @@ logger = logging.getLogger(__name__)
class RSSAnalyser:
async def official_title_parser_movie(
self,
movie: Movie,
rss: RSSItem,
torrent: Torrent,
fetch_poster: bool = True,
):
if not fetch_poster:
pass
elif rss.parser == "mikan":
if not torrent.homepage:
logger.warning("Mikan movie torrent has no homepage info.")
else:
try:
poster_link, official_title = await TitleParser.mikan_parser(
torrent.homepage
)
except AttributeError as e:
logger.warning(
f"Failed to parse Mikan homepage {torrent.homepage}: {e}"
)
else:
movie.poster_link = poster_link
if official_title:
movie.official_title = official_title
elif rss.parser == "tmdb":
tmdb_title, _, year, poster_link = await TitleParser.tmdb_parser(
movie.official_title,
1,
settings.rss_parser.language,
episode_type="movie",
)
movie.official_title = tmdb_title
if year:
try:
movie.year = int(year)
except (ValueError, TypeError):
pass
movie.poster_link = poster_link
if movie.official_title:
movie.official_title = re.sub(r"[/:.\\]", " ", movie.official_title)
async def official_title_parser(
self,
bangumi: Bangumi,
@@ -64,53 +106,86 @@ class RSSAnalyser:
async def torrents_to_data(
self, torrents: list[Torrent], rss: RSSItem, full_parse: bool = True
) -> list:
new_data = []
) -> tuple[list[Bangumi], list[Movie]]:
new_bangumi: list[Bangumi] = []
new_movies: list[Movie] = []
seen_titles: set[str] = set()
for torrent in torrents:
bangumi = await TitleParser.raw_parser(raw=torrent.name)
if bangumi and bangumi.title_raw not in seen_titles:
await self.official_title_parser(
bangumi=bangumi, rss=rss, torrent=torrent
result = await TitleParser.raw_parser(raw=torrent.name)
if result is None or not result.title_raw:
continue
title_raw = result.title_raw
if title_raw in seen_titles:
continue
if isinstance(result, Movie):
await self.official_title_parser_movie(
movie=result, rss=rss, torrent=torrent
)
bangumi.rss_link = rss.url
result.rss_link = rss.url
seen_titles.add(title_raw)
new_movies.append(result)
logger.info(f"New movie found: {result.official_title}")
elif isinstance(result, Bangumi):
await self.official_title_parser(
bangumi=result, rss=rss, torrent=torrent
)
result.rss_link = rss.url
if not full_parse:
return [bangumi]
seen_titles.add(bangumi.title_raw)
new_data.append(bangumi)
logger.info(f"New bangumi founded: {bangumi.official_title}")
return new_data
return [result], new_movies
seen_titles.add(title_raw)
new_bangumi.append(result)
logger.info(f"New bangumi found: {result.official_title}")
return new_bangumi, new_movies
async def torrent_to_data(
self, torrent: Torrent, rss: RSSItem, fetch_poster: bool = True
) -> Bangumi | None:
bangumi = await TitleParser.raw_parser(raw=torrent.name)
if bangumi:
await self.official_title_parser(
bangumi=bangumi, rss=rss, torrent=torrent, fetch_poster=fetch_poster
)
bangumi.rss_link = rss.url
return bangumi
) -> Bangumi | Movie | None:
result = await TitleParser.raw_parser(raw=torrent.name)
if result:
if isinstance(result, Movie):
await self.official_title_parser_movie(
movie=result,
rss=rss,
torrent=torrent,
fetch_poster=fetch_poster,
)
result.rss_link = rss.url
else:
await self.official_title_parser(
bangumi=result,
rss=rss,
torrent=torrent,
fetch_poster=fetch_poster,
)
result.rss_link = rss.url
return result
return None
async def rss_to_data(
self, rss: RSSItem, engine: RSSEngine, full_parse: bool = True
) -> list[Bangumi]:
rss_torrents = await self.get_rss_torrents(rss.url, full_parse)
torrents_to_add = await engine.db.bangumi.match_list(rss_torrents, rss.url)
# Filter out already-known movies first
torrents_after_movies = await engine.db.movie.match_list(rss_torrents, rss.url)
# Then filter out already-known bangumi
torrents_to_add = await engine.db.bangumi.match_list(
torrents_after_movies, rss.url
)
if not torrents_to_add:
logger.debug("No new title has been found.")
return []
# New List
new_data = await self.torrents_to_data(torrents_to_add, rss, full_parse)
if new_data:
# Add to database
await engine.db.bangumi.add_all(new_data)
return new_data
else:
return []
# Parse remaining torrents
new_bangumi, new_movies = await self.torrents_to_data(
torrents_to_add, rss, full_parse
)
for movie in new_movies:
await engine.db.movie.add(movie)
if new_bangumi:
await engine.db.bangumi.add_all(new_bangumi)
return new_bangumi
return []
async def link_to_data(self, rss: RSSItem) -> Bangumi | ResponseModel:
async def link_to_data(self, rss: RSSItem) -> Bangumi | Movie | ResponseModel:
torrents = await self.get_rss_torrents(rss.url, False)
if not torrents:
return ResponseModel(

View File

@@ -10,7 +10,7 @@ from module.conf import settings
from module.database import Database
from module.database.bangumi import _groups_are_similar, match_bangumi_in_list
from module.downloader import AddResult, DownloadClient
from module.models import Bangumi, Episode, ResponseModel, RSSItem, Torrent
from module.models import Bangumi, Episode, Movie, ResponseModel, RSSItem, Torrent
from module.network import RequestContent
from module.notification.events import (
DownloadFailureEvent,
@@ -383,6 +383,46 @@ class RSSEngine:
await self.db.commit()
return events
async def download_movie(self, movie: Movie):
if not movie.rss_link:
return ResponseModel(
status=False,
status_code=406,
msg_en=f"Download movie {movie.official_title} failed: no RSS link.",
msg_zh=f"下载剧场版 {movie.official_title} 失败:缺少 RSS 链接。",
)
async with RequestContent() as req:
filter_pattern = movie.filter.replace(",", "|") if movie.filter else ""
torrents = await req.get_torrents(movie.rss_link, filter_pattern)
if torrents:
async with DownloadClient() as client:
result = await client.add_torrent(
torrents, movie # type: ignore[arg-type]
)
if result is AddResult.FAILED:
return ResponseModel(
status=False,
status_code=502,
msg_en=f"Download movie {movie.official_title} failed.",
msg_zh=f"下载剧场版 {movie.official_title} 失败。",
)
for torrent in torrents:
torrent.downloaded = True
await self.db.torrent.add_all(torrents)
return ResponseModel(
status=True,
status_code=200,
msg_en=f"Download movie {movie.official_title} successfully.",
msg_zh=f"下载剧场版 {movie.official_title} 成功。",
)
else:
return ResponseModel(
status=False,
status_code=406,
msg_en=f"[Engine] Download movie {movie.official_title} failed.",
msg_zh=f"下载剧场版 {movie.official_title} 失败。",
)
async def download_bangumi(self, bangumi: Bangumi):
async with RequestContent() as req:
torrents = await req.get_torrents(

View File

@@ -244,6 +244,60 @@ def _columns(engine, table: str) -> set[str]:
return {c["name"] for c in inspect(engine).get_columns(table)}
def _run_through_version(
engine: Engine,
version: int,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Run the real migration table only through one historical version."""
historical = tuple(item for item in MIGRATIONS if item.version <= version)
with monkeypatch.context() as patch:
patch.setattr(migration_module, "MIGRATIONS", historical)
patch.setattr(migration_module, "CURRENT_SCHEMA_VERSION", version)
run_migrations(engine)
def _make_v18_movie_engine(monkeypatch: pytest.MonkeyPatch) -> Engine:
"""A database produced by the divergent tokenizer dev branch where v18=Movie."""
engine = _make_v0_engine()
_run_through_version(engine, 17, monkeypatch)
with engine.begin() as conn:
conn.execute(text("""CREATE TABLE movie (
id INTEGER PRIMARY KEY,
official_title VARCHAR NOT NULL,
title_raw VARCHAR,
year INTEGER,
group_name VARCHAR,
dpi VARCHAR,
source VARCHAR,
subtitle VARCHAR,
poster_link VARCHAR,
rss_link VARCHAR,
added BOOLEAN NOT NULL DEFAULT 0,
deleted BOOLEAN NOT NULL DEFAULT 0,
save_path VARCHAR,
rule_name VARCHAR,
filter VARCHAR NOT NULL DEFAULT ''
)"""))
conn.execute(text("CREATE INDEX ix_movie_title_raw ON movie(title_raw)"))
conn.execute(text("CREATE INDEX ix_movie_deleted ON movie(deleted)"))
conn.execute(
text(
"INSERT INTO movie "
"(id, official_title, title_raw, year, filter) VALUES "
"(9, 'Preserved Movie', 'Preserved Movie', 2026, '')"
)
)
conn.execute(
text(
"INSERT INTO user (id, username, password) VALUES "
"(7, 'movie_dev_user', 'hashed-password')"
)
)
set_schema_version(conn, 18)
return engine
class TestMigrationTable:
def test_every_migration_has_an_already_applied_guard(self):
"""No migration may rely on the old hardcoded if-ladder."""
@@ -277,6 +331,7 @@ class TestRunMigrations:
assert {"enabled", "created_at", "updated_at"} <= _columns(engine, "user")
assert "auth_session" in inspector.get_table_names()
assert "api_token" in inspector.get_table_names()
assert "movie" in inspector.get_table_names()
user_indexes = {ix["name"] for ix in inspector.get_indexes("user")}
assert "ix_user_username" in user_indexes
@@ -344,6 +399,59 @@ class TestRunMigrations:
indexes = {ix["name"]: ix for ix in inspector.get_indexes("llmcredential")}
assert indexes["ix_llmcredential_provider_id"]["unique"]
def test_upgrades_auth_beta_v20_without_losing_tokens(self, monkeypatch):
engine = _make_v19_auth_engine()
_run_through_version(engine, 20, monkeypatch)
with engine.connect() as conn:
before = conn.execute(
text("SELECT id, token_hash, scope, prefix FROM api_token ORDER BY id")
).all()
run_migrations(engine)
inspector = inspect(engine)
assert "movie" in inspector.get_table_names()
assert {"enabled", "created_at", "updated_at"} <= _columns(engine, "user")
with engine.connect() as conn:
after = conn.execute(
text("SELECT id, token_hash, scope, prefix FROM api_token ORDER BY id")
).all()
assert get_schema_version(conn) == CURRENT_SCHEMA_VERSION
assert after == before
def test_repairs_divergent_movie_v18_database(self, monkeypatch):
engine = _make_v18_movie_engine(monkeypatch)
run_migrations(engine)
inspector = inspect(engine)
assert {"movie", "auth_session", "api_token"} <= set(
inspector.get_table_names()
)
assert {"enabled", "created_at", "updated_at"} <= _columns(engine, "user")
user_indexes = {index["name"] for index in inspector.get_indexes("user")}
assert {"ix_user_username", "ix_user_enabled"} <= user_indexes
token_indexes = {index["name"] for index in inspector.get_indexes("api_token")}
assert "ix_api_token_token_hash_scope" in token_indexes
with engine.connect() as conn:
assert (
conn.execute(
text("SELECT official_title FROM movie WHERE id = 9")
).scalar_one()
== "Preserved Movie"
)
user = conn.execute(
text(
"SELECT username, enabled, created_at, updated_at "
"FROM user WHERE id = 7"
)
).one()
assert user[0] == "movie_dev_user"
assert user[1] == 1
assert user[2] is not None
assert user[3] is not None
assert get_schema_version(conn) == CURRENT_SCHEMA_VERSION
def test_v20_rebuilds_v19_api_tokens_without_losing_data(self):
engine = _make_v19_auth_engine()
# Production upgrade paths may call metadata.create_all before the
@@ -378,7 +486,7 @@ class TestRunMigrations:
.mappings()
.all()
)
assert get_schema_version(conn) == 20
assert get_schema_version(conn) == CURRENT_SCHEMA_VERSION
assert len(after) == len(before) == 2
for old, new in zip(before, after):
@@ -420,7 +528,7 @@ class TestRunMigrations:
def test_v20_rebuild_rolls_back_if_a_late_statement_fails(self, monkeypatch):
engine = _make_v19_auth_engine()
v20 = MIGRATIONS[-1]
v20 = next(item for item in MIGRATIONS if item.version == 20)
broken_v20 = replace(
v20,
statements=(
@@ -432,7 +540,7 @@ class TestRunMigrations:
monkeypatch.setattr(
migration_module,
"MIGRATIONS",
(*MIGRATIONS[:-1], broken_v20),
tuple(broken_v20 if item.version == 20 else item for item in MIGRATIONS),
)
with pytest.raises(OperationalError):
@@ -481,7 +589,7 @@ class TestRunMigrations:
prefix = conn.execute(
text("SELECT prefix FROM api_token WHERE id = 1")
).scalar_one()
assert get_schema_version(conn) == 20
assert get_schema_version(conn) == CURRENT_SCHEMA_VERSION
assert prefix == "legacy_cccccccc"
def test_is_idempotent(self):

View File

@@ -0,0 +1,81 @@
import pytest
from module.parser.analyser import raw_parser
class TestMovieMarkerDetection:
def test_gekijouban_chinese_traditional(self):
content = "[SubGroup] 劇場版 Some Anime Title [1080p]"
info = raw_parser(content)
assert info is not None
assert info.is_movie is True
def test_gekijouban_chinese_simplified(self):
content = "[SubGroup] 剧场版 Some Anime Title [1080p]"
info = raw_parser(content)
assert info is not None
assert info.is_movie is True
def test_movie_keyword_english(self):
content = "[SubGroup] Some Anime Title Movie [1080p HEVC]"
info = raw_parser(content)
assert info is not None
assert info.is_movie is True
def test_gekijouban_romaji(self):
content = "[SubGroup] Gekijouban Some Anime [1080p]"
info = raw_parser(content)
assert info is not None
assert info.is_movie is True
def test_movie_keyword_case_insensitive(self):
content = "[SubGroup] Some Anime MOVIE [720p]"
info = raw_parser(content)
assert info is not None
assert info.is_movie is True
def test_gekijouban_with_hyphen(self):
content = "[SubGroup] Gekijou-ban Title [1080p]"
info = raw_parser(content)
assert info is not None
assert info.is_movie is True
def test_movie_defaults(self):
"""Movies should default to episode=0, season=1."""
content = "[SubGroup] 劇場版 Anime Title [1080p]"
info = raw_parser(content)
assert info is not None
assert info.is_movie is True
assert info.episode == 0
assert info.season == 1
class TestNonMovieRegression:
def test_regular_episode_not_movie(self):
content = "[LoliHouse] Some Anime - 01 [WebRip 1080p HEVC-10bit AAC]"
info = raw_parser(content)
assert info is not None
assert info.is_movie is False
assert info.episode == 1
def test_regular_series_with_season(self):
content = "【幻樱字幕组】【Anime Title S02】【12】【GB_MP4】【1920X1080】"
info = raw_parser(content)
assert info is not None
assert info.is_movie is False
assert info.season == 2
assert info.episode == 12
def test_regular_series_chinese_episode(self):
content = "[字幕组] 动画标题 第5话 [1080p]"
info = raw_parser(content)
assert info is not None
assert info.is_movie is False
assert info.episode == 5
def test_end_marker_not_movie(self):
content = "[LoliHouse] Anime Title - 12 [WebRip 1080p HEVC-10bit AAC][END]"
info = raw_parser(content)
assert info is not None
assert info.is_movie is False
assert info.episode == 12