mirror of
https://github.com/EstrellaXD/Auto_Bangumi.git
synced 2026-10-29 03:36:15 +08:00
feat(parser): add movie/gekijouban support with full DB model
Add detection and handling for anime movies (剧场版/劇場版) as a separate entity from TV series. Previously, movies in RSS feeds were either parsed as episode 0 or skipped entirely. - Add TokenKind.MOVIE and movie keyword/pattern detection in tokenizer - Add is_movie flag to Episode dataclass - Create Movie SQLModel, MovieDatabase with CRUD + TTL cache - Add migration v10 to create movie table - Route TitleParser.raw_parser() to return Movie or Bangumi - Update RSS analyser/engine for movie-aware parsing and downloads - Add movie save path (no Season subfolder) and rename logic - Add Movie REST API endpoints (get/update/delete/disable/enable) - Add 11 tests for movie marker detection and non-movie regression Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -7,6 +7,7 @@ from .downloader import router as downloader_router
|
||||
from .events import router as events_router
|
||||
from .llm import router as llm_router
|
||||
from .log import router as log_router
|
||||
from .movie import router as movie_router
|
||||
from .notification import router as notification_router
|
||||
from .passkey import router as passkey_router
|
||||
from .program import router as program_router
|
||||
@@ -28,6 +29,7 @@ v1.include_router(passkey_router)
|
||||
v1.include_router(log_router)
|
||||
v1.include_router(program_router)
|
||||
v1.include_router(bangumi_router)
|
||||
v1.include_router(movie_router)
|
||||
v1.include_router(config_router)
|
||||
v1.include_router(downloader_router)
|
||||
v1.include_router(events_router)
|
||||
|
||||
100
backend/src/module/api/movie.py
Normal file
100
backend/src/module/api/movie.py
Normal file
@@ -0,0 +1,100 @@
|
||||
from fastapi import APIRouter, Depends
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from module.database import Database, get_db
|
||||
from module.models import APIResponse, Movie
|
||||
from module.models.movie import MovieUpdate
|
||||
from module.security.api import get_current_user
|
||||
|
||||
router = APIRouter(prefix="/movie", tags=["movie"])
|
||||
|
||||
|
||||
@router.get(
|
||||
"/get/all",
|
||||
response_model=list[Movie],
|
||||
dependencies=[Depends(get_current_user)],
|
||||
)
|
||||
async def get_all_movies(db: Database = Depends(get_db)):
|
||||
return await db.movie.search_all()
|
||||
|
||||
|
||||
@router.get(
|
||||
"/get/{movie_id}",
|
||||
response_model=Movie,
|
||||
dependencies=[Depends(get_current_user)],
|
||||
)
|
||||
async def get_movie(movie_id: int, db: Database = Depends(get_db)):
|
||||
movie = await db.movie.search_id(movie_id)
|
||||
if not movie:
|
||||
return JSONResponse(
|
||||
status_code=404,
|
||||
content={
|
||||
"msg_en": f"Movie {movie_id} not found.",
|
||||
"msg_zh": f"未找到剧场版 {movie_id}。",
|
||||
},
|
||||
)
|
||||
return movie
|
||||
|
||||
|
||||
@router.patch(
|
||||
"/update/{movie_id}",
|
||||
response_model=APIResponse,
|
||||
dependencies=[Depends(get_current_user)],
|
||||
)
|
||||
async def update_movie(
|
||||
movie_id: int,
|
||||
data: MovieUpdate,
|
||||
db: Database = Depends(get_db),
|
||||
):
|
||||
success = await db.movie.update(data, _id=movie_id)
|
||||
if success:
|
||||
return JSONResponse(
|
||||
status_code=200,
|
||||
content={"msg_en": "Movie updated.", "msg_zh": "剧场版已更新。"},
|
||||
)
|
||||
return JSONResponse(
|
||||
status_code=404,
|
||||
content={
|
||||
"msg_en": f"Movie {movie_id} not found.",
|
||||
"msg_zh": f"未找到剧场版 {movie_id}。",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/delete/{movie_id}",
|
||||
response_model=APIResponse,
|
||||
dependencies=[Depends(get_current_user)],
|
||||
)
|
||||
async def delete_movie(movie_id: int, db: Database = Depends(get_db)):
|
||||
await db.movie.delete_one(movie_id)
|
||||
return JSONResponse(
|
||||
status_code=200,
|
||||
content={"msg_en": "Movie deleted.", "msg_zh": "剧场版已删除。"},
|
||||
)
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/disable/{movie_id}",
|
||||
response_model=APIResponse,
|
||||
dependencies=[Depends(get_current_user)],
|
||||
)
|
||||
async def disable_movie(movie_id: int, db: Database = Depends(get_db)):
|
||||
await db.movie.disable_rule(movie_id)
|
||||
return JSONResponse(
|
||||
status_code=200,
|
||||
content={"msg_en": "Movie disabled.", "msg_zh": "剧场版已禁用。"},
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/enable/{movie_id}",
|
||||
response_model=APIResponse,
|
||||
dependencies=[Depends(get_current_user)],
|
||||
)
|
||||
async def enable_movie(movie_id: int, db: Database = Depends(get_db)):
|
||||
await db.movie.enable_rule(movie_id)
|
||||
return JSONResponse(
|
||||
status_code=200,
|
||||
content={"msg_en": "Movie enabled.", "msg_zh": "剧场版已启用。"},
|
||||
)
|
||||
@@ -19,6 +19,7 @@ from .migrations import ( # noqa: F401 (re-exported for existing importers)
|
||||
create_tables_async,
|
||||
run_migrations_async,
|
||||
)
|
||||
from .movie import MovieDatabase
|
||||
from .rss import RSSDatabase
|
||||
from .torrent import TorrentDatabase
|
||||
from .user import UserDatabase
|
||||
@@ -44,6 +45,7 @@ class Database:
|
||||
self.rss = RSSDatabase(self.session)
|
||||
self.torrent = TorrentDatabase(self.session)
|
||||
self.bangumi = BangumiDatabase(self.session)
|
||||
self.movie = MovieDatabase(self.session)
|
||||
self.user = UserDatabase(self.session)
|
||||
self.aria2 = Aria2GidDatabase(self.session)
|
||||
self.auth = AuthDatabase(self.session)
|
||||
|
||||
@@ -17,7 +17,7 @@ from pydantic_core import PydanticUndefined
|
||||
from sqlalchemy import Connection, Engine, inspect, text
|
||||
from sqlmodel import SQLModel
|
||||
|
||||
from module.models import ApiToken, AuthSession, Bangumi, User
|
||||
from module.models import ApiToken, AuthSession, Bangumi, Movie, User
|
||||
from module.models.inbox import InboxMessage
|
||||
from module.models.llm_credential import LLMCredential
|
||||
from module.models.passkey import Passkey
|
||||
@@ -29,6 +29,7 @@ logger = logging.getLogger(__name__)
|
||||
# 所有需要进行空值填充的表模型
|
||||
TABLE_MODELS: list[type[SQLModel]] = [
|
||||
Bangumi,
|
||||
Movie,
|
||||
RSSItem,
|
||||
Torrent,
|
||||
User,
|
||||
@@ -512,6 +513,110 @@ MIGRATIONS: tuple[Migration, ...] = (
|
||||
# every database still below v20 must run the transactional rebuild.
|
||||
lambda _inspector: False,
|
||||
),
|
||||
Migration(
|
||||
21,
|
||||
"create movie table for standalone movie subscriptions",
|
||||
(
|
||||
"""CREATE TABLE IF NOT EXISTS movie (
|
||||
id INTEGER PRIMARY KEY,
|
||||
official_title VARCHAR NOT NULL,
|
||||
title_raw VARCHAR,
|
||||
year INTEGER,
|
||||
group_name VARCHAR,
|
||||
dpi VARCHAR,
|
||||
source VARCHAR,
|
||||
subtitle VARCHAR,
|
||||
poster_link VARCHAR,
|
||||
rss_link VARCHAR,
|
||||
added BOOLEAN NOT NULL DEFAULT 0,
|
||||
deleted BOOLEAN NOT NULL DEFAULT 0,
|
||||
save_path VARCHAR,
|
||||
rule_name VARCHAR,
|
||||
filter VARCHAR NOT NULL DEFAULT ''
|
||||
)""",
|
||||
"CREATE INDEX IF NOT EXISTS ix_movie_title_raw ON movie(title_raw)",
|
||||
"CREATE INDEX IF NOT EXISTS ix_movie_deleted ON movie(deleted)",
|
||||
),
|
||||
all_checks(
|
||||
table_exists("movie"),
|
||||
index_exists("movie", "ix_movie_title_raw"),
|
||||
index_exists("movie", "ix_movie_deleted"),
|
||||
),
|
||||
(
|
||||
(
|
||||
"""CREATE TABLE IF NOT EXISTS movie (
|
||||
id INTEGER PRIMARY KEY,
|
||||
official_title VARCHAR NOT NULL,
|
||||
title_raw VARCHAR,
|
||||
year INTEGER,
|
||||
group_name VARCHAR,
|
||||
dpi VARCHAR,
|
||||
source VARCHAR,
|
||||
subtitle VARCHAR,
|
||||
poster_link VARCHAR,
|
||||
rss_link VARCHAR,
|
||||
added BOOLEAN NOT NULL DEFAULT 0,
|
||||
deleted BOOLEAN NOT NULL DEFAULT 0,
|
||||
save_path VARCHAR,
|
||||
rule_name VARCHAR,
|
||||
filter VARCHAR NOT NULL DEFAULT ''
|
||||
)""",
|
||||
table_exists("movie"),
|
||||
),
|
||||
(
|
||||
"CREATE INDEX IF NOT EXISTS ix_movie_title_raw ON movie(title_raw)",
|
||||
index_exists("movie", "ix_movie_title_raw"),
|
||||
),
|
||||
(
|
||||
"CREATE INDEX IF NOT EXISTS ix_movie_deleted ON movie(deleted)",
|
||||
index_exists("movie", "ix_movie_deleted"),
|
||||
),
|
||||
),
|
||||
),
|
||||
Migration(
|
||||
22,
|
||||
"repair multi-user v18 fields after the divergent movie development schema",
|
||||
(
|
||||
"ALTER TABLE user ADD COLUMN enabled BOOLEAN NOT NULL DEFAULT 1",
|
||||
"ALTER TABLE user ADD COLUMN created_at TIMESTAMP NOT NULL "
|
||||
"DEFAULT '1970-01-01 00:00:00'",
|
||||
"ALTER TABLE user ADD COLUMN updated_at TIMESTAMP NOT NULL "
|
||||
"DEFAULT '1970-01-01 00:00:00'",
|
||||
"CREATE UNIQUE INDEX IF NOT EXISTS ix_user_username ON user(username)",
|
||||
"CREATE INDEX IF NOT EXISTS ix_user_enabled ON user(enabled)",
|
||||
),
|
||||
all_checks(
|
||||
column_exists("user", "enabled"),
|
||||
column_exists("user", "created_at"),
|
||||
column_exists("user", "updated_at"),
|
||||
index_exists("user", "ix_user_username"),
|
||||
index_exists("user", "ix_user_enabled"),
|
||||
),
|
||||
(
|
||||
(
|
||||
"ALTER TABLE user ADD COLUMN enabled BOOLEAN NOT NULL DEFAULT 1",
|
||||
column_exists("user", "enabled"),
|
||||
),
|
||||
(
|
||||
"ALTER TABLE user ADD COLUMN created_at TIMESTAMP NOT NULL "
|
||||
"DEFAULT '1970-01-01 00:00:00'",
|
||||
column_exists("user", "created_at"),
|
||||
),
|
||||
(
|
||||
"ALTER TABLE user ADD COLUMN updated_at TIMESTAMP NOT NULL "
|
||||
"DEFAULT '1970-01-01 00:00:00'",
|
||||
column_exists("user", "updated_at"),
|
||||
),
|
||||
(
|
||||
"CREATE UNIQUE INDEX IF NOT EXISTS ix_user_username ON user(username)",
|
||||
index_exists("user", "ix_user_username"),
|
||||
),
|
||||
(
|
||||
"CREATE INDEX IF NOT EXISTS ix_user_enabled ON user(enabled)",
|
||||
index_exists("user", "ix_user_enabled"),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
# 由迁移列表派生,新增迁移时无需手动同步
|
||||
|
||||
171
backend/src/module/database/movie.py
Normal file
171
backend/src/module/database/movie.py
Normal file
@@ -0,0 +1,171 @@
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlmodel import and_, false, select
|
||||
|
||||
from module.models.movie import Movie, MovieUpdate
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Module-level TTL cache for search_all results
|
||||
_movie_cache: list[Movie] | None = None
|
||||
_movie_cache_time: float = 0
|
||||
_MOVIE_CACHE_TTL: float = 300.0
|
||||
|
||||
|
||||
def _invalidate_movie_cache():
|
||||
global _movie_cache, _movie_cache_time
|
||||
_movie_cache = None
|
||||
_movie_cache_time = 0
|
||||
|
||||
|
||||
class MovieDatabase:
|
||||
def __init__(self, session: AsyncSession):
|
||||
self.session = session
|
||||
|
||||
async def _is_duplicate(self, data: Movie) -> bool:
|
||||
statement = select(Movie).where(
|
||||
and_(
|
||||
Movie.title_raw == data.title_raw,
|
||||
Movie.group_name == data.group_name,
|
||||
)
|
||||
)
|
||||
result = await self.session.execute(statement)
|
||||
return result.scalar_one_or_none() is not None
|
||||
|
||||
async def add(self, data: Movie) -> bool:
|
||||
if await self._is_duplicate(data):
|
||||
logger.debug(
|
||||
"Skipping duplicate movie: %s (%s)",
|
||||
data.official_title,
|
||||
data.group_name,
|
||||
)
|
||||
return False
|
||||
self.session.add(data)
|
||||
await self.session.commit()
|
||||
_invalidate_movie_cache()
|
||||
logger.debug("Insert movie %s into database.", data.official_title)
|
||||
return True
|
||||
|
||||
async def search_all(self) -> list[Movie]:
|
||||
global _movie_cache, _movie_cache_time
|
||||
now = time.time()
|
||||
if _movie_cache is not None and (now - _movie_cache_time) < _MOVIE_CACHE_TTL:
|
||||
return _movie_cache
|
||||
statement = select(Movie)
|
||||
result = await self.session.execute(statement)
|
||||
movies = list(result.scalars().all())
|
||||
for m in movies:
|
||||
self.session.expunge(m)
|
||||
_movie_cache = movies
|
||||
_movie_cache_time = now
|
||||
return _movie_cache
|
||||
|
||||
async def search_id(self, _id: int) -> Optional[Movie]:
|
||||
statement = select(Movie).where(Movie.id == _id)
|
||||
result = await self.session.execute(statement)
|
||||
movie = result.scalar_one_or_none()
|
||||
if movie is None:
|
||||
logger.warning(f"Cannot find movie id: {_id}.")
|
||||
return None
|
||||
return movie
|
||||
|
||||
async def update(self, data: Movie | MovieUpdate, _id: int | None = None) -> bool:
|
||||
if _id and isinstance(data, MovieUpdate):
|
||||
db_data = await self.session.get(Movie, _id)
|
||||
elif isinstance(data, Movie):
|
||||
db_data = await self.session.get(Movie, data.id)
|
||||
else:
|
||||
return False
|
||||
if not db_data:
|
||||
return False
|
||||
movie_data = data.model_dump(exclude_unset=True)
|
||||
for key, value in movie_data.items():
|
||||
setattr(db_data, key, value)
|
||||
self.session.add(db_data)
|
||||
await self.session.commit()
|
||||
_invalidate_movie_cache()
|
||||
logger.debug("Update movie %s", data.official_title)
|
||||
return True
|
||||
|
||||
async def delete_one(self, _id: int):
|
||||
statement = select(Movie).where(Movie.id == _id)
|
||||
result = await self.session.execute(statement)
|
||||
movie = result.scalar_one_or_none()
|
||||
if movie:
|
||||
await self.session.delete(movie)
|
||||
await self.session.commit()
|
||||
_invalidate_movie_cache()
|
||||
logger.debug("Delete movie id: %s.", _id)
|
||||
|
||||
async def disable_rule(self, _id: int):
|
||||
statement = select(Movie).where(Movie.id == _id)
|
||||
result = await self.session.execute(statement)
|
||||
movie = result.scalar_one_or_none()
|
||||
if movie:
|
||||
movie.deleted = True
|
||||
self.session.add(movie)
|
||||
await self.session.commit()
|
||||
_invalidate_movie_cache()
|
||||
logger.debug("Disable movie %s.", movie.title_raw)
|
||||
|
||||
async def enable_rule(self, _id: int):
|
||||
statement = select(Movie).where(Movie.id == _id)
|
||||
result = await self.session.execute(statement)
|
||||
movie = result.scalar_one_or_none()
|
||||
if movie:
|
||||
movie.deleted = False
|
||||
self.session.add(movie)
|
||||
await self.session.commit()
|
||||
_invalidate_movie_cache()
|
||||
logger.debug("Enable movie %s.", movie.title_raw)
|
||||
|
||||
async def match_list(self, torrent_list: list, rss_link: str) -> list:
|
||||
match_datas = await self.search_all()
|
||||
if not match_datas:
|
||||
return torrent_list
|
||||
|
||||
title_index: dict[str, Movie] = {}
|
||||
for m in match_datas:
|
||||
if m.title_raw:
|
||||
title_index[m.title_raw] = m
|
||||
|
||||
if not title_index:
|
||||
return torrent_list
|
||||
|
||||
sorted_titles = sorted(title_index.keys(), key=len, reverse=True)
|
||||
pattern = "|".join(re.escape(title) for title in sorted_titles)
|
||||
title_regex = re.compile(pattern)
|
||||
|
||||
unmatched = []
|
||||
rss_updated = set()
|
||||
for torrent in torrent_list:
|
||||
match = title_regex.search(torrent.name)
|
||||
if match:
|
||||
matched_title = match.group(0)
|
||||
match_data = title_index[matched_title]
|
||||
if (
|
||||
match_data.rss_link
|
||||
and rss_link not in match_data.rss_link
|
||||
and match_data.title_raw not in rss_updated
|
||||
):
|
||||
match_data = await self.session.merge(match_data)
|
||||
match_data.rss_link = f"{match_data.rss_link},{rss_link}"
|
||||
match_data.added = False
|
||||
rss_updated.add(match_data.title_raw)
|
||||
else:
|
||||
unmatched.append(torrent)
|
||||
if rss_updated:
|
||||
await self.session.commit()
|
||||
_invalidate_movie_cache()
|
||||
return unmatched
|
||||
|
||||
async def not_added(self) -> list[Movie]:
|
||||
conditions = select(Movie).where(
|
||||
Movie.added == false(),
|
||||
)
|
||||
result = await self.session.execute(conditions)
|
||||
return list(result.scalars().all())
|
||||
@@ -5,6 +5,7 @@ from pathlib import PureWindowsPath
|
||||
|
||||
from module.conf import PLATFORM, settings
|
||||
from module.models import Bangumi, BangumiUpdate
|
||||
from module.models.movie import Movie, MovieUpdate
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -77,7 +78,17 @@ def is_ep(file_path: PathLike[str] | str):
|
||||
return file_depth(file_path) <= 2
|
||||
|
||||
|
||||
def gen_save_path(data: Bangumi | BangumiUpdate):
|
||||
def _media_folder(data: Bangumi | BangumiUpdate | Movie | MovieUpdate) -> str:
|
||||
title = data.official_title or "Unknown Bangumi"
|
||||
folder = sanitize_path_fragment(f"{title} ({data.year})" if data.year else title)
|
||||
if folder:
|
||||
return folder
|
||||
# 标题全由保留字符组成时清洗结果为空——不能让所有这类条目
|
||||
# 坍缩到同一个下载目录里
|
||||
return "Unknown Bangumi"
|
||||
|
||||
|
||||
def gen_save_path(data: Bangumi | BangumiUpdate | Movie | MovieUpdate) -> str:
|
||||
"""Generate save path for a bangumi.
|
||||
|
||||
The save path uses the adjusted season number (season + season_offset)
|
||||
@@ -87,15 +98,9 @@ def gen_save_path(data: Bangumi | BangumiUpdate):
|
||||
specials/OVA/OAD land in "Season 0" (Jellyfin/Plex convention) instead of
|
||||
being interleaved with regular episodes.
|
||||
"""
|
||||
folder = sanitize_path_fragment(
|
||||
f"{data.official_title} ({data.year})" if data.year else data.official_title
|
||||
)
|
||||
if not folder:
|
||||
# 标题全由保留字符组成时清洗结果为空——不能让所有这类番剧
|
||||
# 坍缩到同一个 ``…//Season N`` 目录里
|
||||
folder = "Unknown Bangumi"
|
||||
folder = _media_folder(data)
|
||||
episode_type = getattr(data, "episode_type", "episode")
|
||||
if episode_type == "movie":
|
||||
if isinstance(data, (Movie, MovieUpdate)) or episode_type == "movie":
|
||||
# 电影/剧场版:Title (Year)/Title (Year).ext,不建 Season 子目录
|
||||
return str(Path(settings.downloader.path) / folder)
|
||||
# Apply season_offset to get the adjusted season number for the folder
|
||||
@@ -112,6 +117,19 @@ def gen_save_path(data: Bangumi | BangumiUpdate):
|
||||
return str(save_path)
|
||||
|
||||
|
||||
def gen_movie_save_path(data: Movie | MovieUpdate) -> str:
|
||||
"""Generate the flat save directory used by a movie/gekijouban."""
|
||||
return str(Path(settings.downloader.path) / _media_folder(data))
|
||||
|
||||
|
||||
def movie_rule_name(data: Movie) -> str:
|
||||
return (
|
||||
f"[{data.group_name}] {data.official_title}"
|
||||
if settings.bangumi_manage.group_tag
|
||||
else data.official_title
|
||||
)
|
||||
|
||||
|
||||
def rule_name(data: Bangumi):
|
||||
name = (
|
||||
f"[{data.group_name}] {data.official_title} S{data.season}"
|
||||
|
||||
@@ -85,6 +85,16 @@ class Renamer:
|
||||
episode = int(episode)
|
||||
return f"0{episode}" if episode < 10 else str(episode)
|
||||
|
||||
@staticmethod
|
||||
def gen_movie_path(
|
||||
file_info: EpisodeFile | SubtitleFile,
|
||||
movie_name: str,
|
||||
method: str,
|
||||
) -> str:
|
||||
if method in ("none", "subtitle_none"):
|
||||
return file_info.media_path
|
||||
return f"{movie_name}{file_info.suffix}"
|
||||
|
||||
@staticmethod
|
||||
def gen_path(
|
||||
file_info: EpisodeFile | SubtitleFile,
|
||||
@@ -237,6 +247,35 @@ class Renamer:
|
||||
stem = stem[len(prefix) :]
|
||||
return f"{base} - {stem}{suffix}"
|
||||
|
||||
async def rename_movie_file(
|
||||
self,
|
||||
torrent_name: str,
|
||||
media_path: str,
|
||||
movie_name: str,
|
||||
method: str,
|
||||
_hash: str,
|
||||
**kwargs,
|
||||
):
|
||||
ep = self._parser.torrent_parser(
|
||||
torrent_name=torrent_name,
|
||||
torrent_path=media_path,
|
||||
episode_type="movie",
|
||||
)
|
||||
if ep:
|
||||
new_path = self.gen_movie_path(ep, movie_name, method=method)
|
||||
if media_path != new_path:
|
||||
if await self.client.rename_torrent_file(
|
||||
_hash=_hash, old_path=media_path, new_path=new_path
|
||||
):
|
||||
return Notification(
|
||||
official_title=movie_name,
|
||||
season=0,
|
||||
episode=0,
|
||||
)
|
||||
else:
|
||||
logger.warning(f"{media_path} parse failed (movie)")
|
||||
return None
|
||||
|
||||
async def rename_collection(
|
||||
self,
|
||||
media_list: list[str],
|
||||
|
||||
@@ -3,6 +3,7 @@ from .bangumi import Bangumi, BangumiUpdate, Episode, Notification
|
||||
from .config import Config
|
||||
from .inbox import InboxMessage
|
||||
from .llm_credential import LLMCredential
|
||||
from .movie import Movie, MovieUpdate
|
||||
from .passkey import Passkey, PasskeyCreate, PasskeyDelete, PasskeyList
|
||||
from .response import APIResponse, ResponseModel
|
||||
from .rss import RSSItem, RSSUpdate
|
||||
|
||||
@@ -131,6 +131,14 @@ class Episode:
|
||||
resolution: str
|
||||
source: str
|
||||
episode_type: str = "episode" # "episode" | "movie" | "special"
|
||||
is_movie: bool = False
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""Keep the legacy movie flag and the typed classifier in sync."""
|
||||
if self.is_movie:
|
||||
self.episode_type = "movie"
|
||||
elif self.episode_type == "movie":
|
||||
self.is_movie = True
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
|
||||
41
backend/src/module/models/movie.py
Normal file
41
backend/src/module/models/movie.py
Normal file
@@ -0,0 +1,41 @@
|
||||
from typing import Optional
|
||||
|
||||
from sqlmodel import Field, SQLModel
|
||||
|
||||
|
||||
class Movie(SQLModel, table=True):
|
||||
id: int = Field(default=None, primary_key=True)
|
||||
official_title: str = Field(
|
||||
default="official_title", alias="official_title", title="剧场版名"
|
||||
)
|
||||
title_raw: str | None = Field(
|
||||
default=None, alias="title_raw", title="剧场版原名", index=True
|
||||
)
|
||||
year: int | None = Field(default=None, alias="year", title="年份")
|
||||
group_name: str | None = Field(default=None, alias="group_name", title="字幕组")
|
||||
dpi: str | None = Field(default=None, alias="dpi", title="分辨率")
|
||||
source: str | None = Field(default=None, alias="source", title="来源")
|
||||
subtitle: str | None = Field(default=None, alias="subtitle", title="字幕")
|
||||
poster_link: str | None = Field(default=None, alias="poster_link", title="海报链接")
|
||||
rss_link: str | None = Field(default=None, alias="rss_link", title="RSS链接")
|
||||
added: bool = Field(default=False, alias="added", title="是否已添加")
|
||||
deleted: bool = Field(
|
||||
default=False, alias="deleted", title="是否已删除", index=True
|
||||
)
|
||||
save_path: str | None = Field(default=None, alias="save_path", title="保存路径")
|
||||
rule_name: str | None = Field(default=None, alias="rule_name", title="规则名")
|
||||
filter: str = Field(default="", alias="filter", title="过滤器")
|
||||
|
||||
|
||||
class MovieUpdate(SQLModel):
|
||||
official_title: str | None = None
|
||||
title_raw: str | None = None
|
||||
year: int | None = None
|
||||
group_name: str | None = None
|
||||
dpi: str | None = None
|
||||
source: str | None = None
|
||||
subtitle: str | None = None
|
||||
poster_link: str | None = None
|
||||
rss_link: str | None = None
|
||||
save_path: str | None = None
|
||||
filter: str | None = None
|
||||
121
backend/src/module/parser/analyser/tokenizer/keywords.py
Normal file
121
backend/src/module/parser/analyser/tokenizer/keywords.py
Normal file
@@ -0,0 +1,121 @@
|
||||
import re
|
||||
|
||||
from .token import TokenKind
|
||||
|
||||
KEYWORDS: dict[str, TokenKind] = {
|
||||
# Resolution
|
||||
"1080p": TokenKind.RESOLUTION,
|
||||
"720p": TokenKind.RESOLUTION,
|
||||
"2160p": TokenKind.RESOLUTION,
|
||||
"4k": TokenKind.RESOLUTION,
|
||||
"1080": TokenKind.RESOLUTION,
|
||||
"720": TokenKind.RESOLUTION,
|
||||
"2160": TokenKind.RESOLUTION,
|
||||
"1920x1080": TokenKind.RESOLUTION,
|
||||
"3840x2160": TokenKind.RESOLUTION,
|
||||
# Source
|
||||
"baha": TokenKind.SOURCE,
|
||||
"b-global": TokenKind.SOURCE,
|
||||
"bilibili": TokenKind.SOURCE,
|
||||
"at-x": TokenKind.SOURCE,
|
||||
"web-dl": TokenKind.SOURCE,
|
||||
"webrip": TokenKind.SOURCE,
|
||||
"webdl": TokenKind.SOURCE,
|
||||
"web": TokenKind.SOURCE,
|
||||
"web-dl remux": TokenKind.SOURCE,
|
||||
"ttfc": TokenKind.SOURCE,
|
||||
"cr": TokenKind.SOURCE,
|
||||
"adn": TokenKind.SOURCE,
|
||||
"iqiyi": TokenKind.SOURCE,
|
||||
"abema": TokenKind.SOURCE,
|
||||
# Codec
|
||||
"hevc": TokenKind.CODEC,
|
||||
"avc": TokenKind.CODEC,
|
||||
"x264": TokenKind.CODEC,
|
||||
"x265": TokenKind.CODEC,
|
||||
"hevc-10bit": TokenKind.CODEC,
|
||||
"hevc10-bit": TokenKind.CODEC,
|
||||
"h264": TokenKind.CODEC,
|
||||
"h265": TokenKind.CODEC,
|
||||
"h.264": TokenKind.CODEC,
|
||||
"h.265": TokenKind.CODEC,
|
||||
"10bit": TokenKind.CODEC,
|
||||
# Audio
|
||||
"aac": TokenKind.AUDIO,
|
||||
"flac": TokenKind.AUDIO,
|
||||
"mp3": TokenKind.AUDIO,
|
||||
"ac3": TokenKind.AUDIO,
|
||||
"dts": TokenKind.AUDIO,
|
||||
# Container
|
||||
"mp4": TokenKind.CONTAINER,
|
||||
"mkv": TokenKind.CONTAINER,
|
||||
"avi": TokenKind.CONTAINER,
|
||||
# Subtitle markers
|
||||
"cht": TokenKind.SUBTITLE,
|
||||
"chs": TokenKind.SUBTITLE,
|
||||
"big5": TokenKind.SUBTITLE,
|
||||
"gb": TokenKind.SUBTITLE,
|
||||
"assx2": TokenKind.SUBTITLE,
|
||||
"multi sub": TokenKind.SUBTITLE,
|
||||
"vostfr": TokenKind.SUBTITLE,
|
||||
# Movie markers
|
||||
"movie": TokenKind.MOVIE,
|
||||
"gekijouban": TokenKind.MOVIE,
|
||||
"the movie": TokenKind.MOVIE,
|
||||
# Version
|
||||
"v2": TokenKind.VERSION,
|
||||
"v3": TokenKind.VERSION,
|
||||
# End marker
|
||||
"end": TokenKind.END_MARKER,
|
||||
}
|
||||
|
||||
|
||||
PATTERN_CHECKS: list[tuple[re.Pattern, TokenKind]] = [
|
||||
# Episode patterns
|
||||
(re.compile(r"^[Ee][Pp]?\d+$"), TokenKind.EPISODE),
|
||||
(re.compile(r"^第?\d+[话話集]$"), TokenKind.EPISODE),
|
||||
(re.compile(r"^\d+[Pp]$"), TokenKind.RESOLUTION),
|
||||
(re.compile(r"^\d{3,4}[xX]\d{3,4}$"), TokenKind.RESOLUTION),
|
||||
# Season patterns
|
||||
(re.compile(r"^[Ss]\d{1,2}$"), TokenKind.SEASON),
|
||||
(re.compile(r"^Season \d+$", re.I), TokenKind.SEASON),
|
||||
(re.compile(r"^第[一二三四五六七八九十\d]+[季期]$"), TokenKind.SEASON),
|
||||
# S01E05 combined pattern
|
||||
(re.compile(r"^[Ss]\d{1,2}[Ee]\d+$"), TokenKind.EPISODE),
|
||||
# Subtitle patterns
|
||||
(re.compile(r"^[简繁日字幕][简繁日字幕体文]*"), TokenKind.SUBTITLE),
|
||||
(re.compile(r"^GB_?(?:JP|MP4|MKV)?$", re.I), TokenKind.SUBTITLE),
|
||||
(re.compile(r"^CHS_JP$", re.I), TokenKind.SUBTITLE),
|
||||
(re.compile(r"^CHT_JP$", re.I), TokenKind.SUBTITLE),
|
||||
# Prefix tags
|
||||
(re.compile(r"^\d+月新番$"), TokenKind.PREFIX_TAG),
|
||||
(re.compile(r"^新番$"), TokenKind.PREFIX_TAG),
|
||||
(re.compile(r"^★.*★$"), TokenKind.PREFIX_TAG),
|
||||
# Version pattern
|
||||
(re.compile(r"^[vV]\d$"), TokenKind.VERSION),
|
||||
# Hash (CRC32)
|
||||
(re.compile(r"^[0-9A-Fa-f]{8}$"), TokenKind.HASH),
|
||||
# Movie markers
|
||||
(re.compile(r"^[剧劇][场場][版]?$|^劇場版?$"), TokenKind.MOVIE),
|
||||
(re.compile(r"^Movie$", re.I), TokenKind.MOVIE),
|
||||
(re.compile(r"^Gekijou-?ban$", re.I), TokenKind.MOVIE),
|
||||
# Date pattern
|
||||
(re.compile(r"^\d{4}\.\d{2}\.\d{2}$"), TokenKind.EXTRA),
|
||||
# File size
|
||||
(re.compile(r"^\d+\.\d+ [KMGT]B$", re.I), TokenKind.EXTRA),
|
||||
]
|
||||
|
||||
CHINESE_NUMBER_MAP: dict[str, int] = {
|
||||
"一": 1,
|
||||
"二": 2,
|
||||
"三": 3,
|
||||
"四": 4,
|
||||
"五": 5,
|
||||
"六": 6,
|
||||
"七": 7,
|
||||
"八": 8,
|
||||
"九": 9,
|
||||
"十": 10,
|
||||
}
|
||||
|
||||
FILE_EXTENSIONS = re.compile(r"\.(mp4|mkv|avi|ass|srt|ssa|sub)$", re.I)
|
||||
100
backend/src/module/parser/analyser/tokenizer/stage_compose.py
Normal file
100
backend/src/module/parser/analyser/tokenizer/stage_compose.py
Normal file
@@ -0,0 +1,100 @@
|
||||
import re
|
||||
|
||||
from module.models import Episode
|
||||
|
||||
from .keywords import CHINESE_NUMBER_MAP
|
||||
from .token import Token, TokenKind
|
||||
|
||||
# Only strip dashes that have surrounding whitespace (separators),
|
||||
# not dashes attached to characters (part of titles like 地。-关于地球的运动-)
|
||||
_DASH_STRIP = re.compile(r"^\s*-\s+|\s+-\s*$")
|
||||
_SEASON_NUM = re.compile(r"[Ss](\d+)|Season\s+(\d+)", re.I)
|
||||
_CHINESE_SEASON = re.compile(r"第([一二三四五六七八九十\d]+)[季期]")
|
||||
_SUB_CLEAN = re.compile(r"_MP4|_MKV", re.I)
|
||||
_CJK_SUB = re.compile(r"[简繁日字幕]")
|
||||
|
||||
|
||||
def compose(tokens: list[Token]) -> Episode | None:
|
||||
title_en: str | None = None
|
||||
title_zh: str | None = None
|
||||
title_jp: str | None = None
|
||||
season = 1
|
||||
season_raw = ""
|
||||
episode = 0
|
||||
sub: str | None = None
|
||||
group = ""
|
||||
resolution: str | None = None
|
||||
source: str | None = None
|
||||
|
||||
is_movie = False
|
||||
|
||||
# Collect all subtitles, prefer CJK ones
|
||||
all_subs: list[str] = []
|
||||
|
||||
for token in tokens:
|
||||
kind = token.kind
|
||||
text = token.text
|
||||
|
||||
if kind == TokenKind.GROUP:
|
||||
group = text
|
||||
elif kind == TokenKind.TITLE_ZH:
|
||||
text = _DASH_STRIP.sub("", text).strip()
|
||||
if text:
|
||||
title_zh = text
|
||||
elif kind == TokenKind.TITLE_EN:
|
||||
title_en = text.strip()
|
||||
elif kind == TokenKind.TITLE_JP:
|
||||
title_jp = text.strip()
|
||||
elif kind == TokenKind.EPISODE:
|
||||
ep_digits = re.search(r"\d+", text)
|
||||
if ep_digits:
|
||||
episode = int(ep_digits.group())
|
||||
elif kind == TokenKind.SEASON:
|
||||
m = _SEASON_NUM.match(text)
|
||||
if m:
|
||||
s_num = int(m.group(1) or m.group(2))
|
||||
else:
|
||||
cm = _CHINESE_SEASON.match(text)
|
||||
if cm:
|
||||
s_val = cm.group(1)
|
||||
try:
|
||||
s_num = int(s_val)
|
||||
except ValueError:
|
||||
s_num = CHINESE_NUMBER_MAP.get(s_val, 1)
|
||||
else:
|
||||
s_num = season
|
||||
# Prefer first season found (Chinese season over English)
|
||||
if season == 1 or not season_raw:
|
||||
season = s_num
|
||||
season_raw = text
|
||||
elif kind == TokenKind.RESOLUTION:
|
||||
if resolution is None:
|
||||
resolution = text
|
||||
elif kind == TokenKind.SOURCE:
|
||||
if source is None:
|
||||
source = text
|
||||
elif kind == TokenKind.MOVIE:
|
||||
is_movie = True
|
||||
elif kind == TokenKind.SUBTITLE:
|
||||
cleaned = _SUB_CLEAN.sub("", text)
|
||||
if cleaned:
|
||||
all_subs.append(cleaned)
|
||||
|
||||
# Prefer CJK subtitle markers over technical ones
|
||||
if all_subs:
|
||||
cjk_subs = [s for s in all_subs if _CJK_SUB.search(s)]
|
||||
sub = cjk_subs[0] if cjk_subs else all_subs[0]
|
||||
|
||||
return Episode(
|
||||
title_en=title_en,
|
||||
title_zh=title_zh,
|
||||
title_jp=title_jp,
|
||||
season=season,
|
||||
season_raw=season_raw,
|
||||
episode=episode,
|
||||
sub=sub,
|
||||
group=group,
|
||||
resolution=resolution,
|
||||
source=source,
|
||||
is_movie=is_movie,
|
||||
)
|
||||
32
backend/src/module/parser/analyser/tokenizer/token.py
Normal file
32
backend/src/module/parser/analyser/tokenizer/token.py
Normal file
@@ -0,0 +1,32 @@
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
|
||||
|
||||
class TokenKind(Enum):
|
||||
UNKNOWN = auto()
|
||||
GROUP = auto()
|
||||
TITLE_ZH = auto()
|
||||
TITLE_EN = auto()
|
||||
TITLE_JP = auto()
|
||||
EPISODE = auto()
|
||||
SEASON = auto()
|
||||
RESOLUTION = auto()
|
||||
SOURCE = auto()
|
||||
SUBTITLE = auto()
|
||||
CODEC = auto()
|
||||
AUDIO = auto()
|
||||
CONTAINER = auto()
|
||||
PREFIX_TAG = auto()
|
||||
VERSION = auto()
|
||||
END_MARKER = auto()
|
||||
HASH = auto()
|
||||
MOVIE = auto() # movie/gekijouban marker
|
||||
EXTRA = auto()
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Token:
|
||||
text: str
|
||||
kind: TokenKind
|
||||
position: int
|
||||
enclosed: bool
|
||||
@@ -3,7 +3,7 @@ import logging
|
||||
import time
|
||||
|
||||
from module.conf import settings
|
||||
from module.models import Bangumi
|
||||
from module.models import Bangumi, Movie
|
||||
from module.models.bangumi import Episode
|
||||
from module.models.config import LLM
|
||||
from module.parser.analyser import (
|
||||
@@ -254,8 +254,62 @@ class TitleParser:
|
||||
logger.warning("Please change bangumi info manually.")
|
||||
|
||||
@staticmethod
|
||||
async def raw_parser(raw: str) -> Bangumi | None:
|
||||
def _resolve_titles(episode: Episode) -> tuple[str | None, str | None]:
|
||||
"""Extract official_title and title_raw from an Episode."""
|
||||
language = settings.rss_parser.language
|
||||
titles = {
|
||||
"zh": episode.title_zh,
|
||||
"en": episode.title_en,
|
||||
"jp": episode.title_jp,
|
||||
}
|
||||
title_raw = episode.title_en or episode.title_zh or episode.title_jp
|
||||
if titles[language]:
|
||||
official_title = titles[language]
|
||||
elif titles["zh"]:
|
||||
official_title = titles["zh"]
|
||||
elif titles["en"]:
|
||||
official_title = titles["en"]
|
||||
elif titles["jp"]:
|
||||
official_title = titles["jp"]
|
||||
else:
|
||||
official_title = title_raw
|
||||
return official_title, title_raw
|
||||
|
||||
@staticmethod
|
||||
def _episode_to_bangumi(
|
||||
episode: Episode, official_title: str, title_raw: str
|
||||
) -> Bangumi:
|
||||
return Bangumi(
|
||||
official_title=official_title,
|
||||
title_raw=title_raw,
|
||||
season=episode.season,
|
||||
season_raw=episode.season_raw,
|
||||
group_name=episode.group,
|
||||
dpi=episode.resolution,
|
||||
source=episode.source,
|
||||
subtitle=episode.sub,
|
||||
eps_collect=False if episode.episode > 1 else True,
|
||||
episode_offset=0,
|
||||
filter=",".join(settings.rss_parser.filter),
|
||||
episode_type=episode.episode_type,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _episode_to_movie(
|
||||
episode: Episode, official_title: str, title_raw: str
|
||||
) -> Movie:
|
||||
return Movie(
|
||||
official_title=official_title,
|
||||
title_raw=title_raw,
|
||||
group_name=episode.group,
|
||||
dpi=episode.resolution,
|
||||
source=episode.source,
|
||||
subtitle=episode.sub,
|
||||
filter=",".join(settings.rss_parser.filter),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def raw_parser(raw: str) -> Bangumi | Movie | None:
|
||||
try:
|
||||
llm_conf = _llm_config()
|
||||
episode = None
|
||||
@@ -280,41 +334,21 @@ class TitleParser:
|
||||
if detected_episode_type is not None:
|
||||
episode.episode_type = detected_episode_type
|
||||
|
||||
titles = {
|
||||
"zh": episode.title_zh,
|
||||
"en": episode.title_en,
|
||||
"jp": episode.title_jp,
|
||||
}
|
||||
title_raw = episode.title_en or episode.title_zh or episode.title_jp
|
||||
if titles[language]:
|
||||
official_title = titles[language]
|
||||
elif titles["zh"]:
|
||||
official_title = titles["zh"]
|
||||
elif titles["en"]:
|
||||
official_title = titles["en"]
|
||||
elif titles["jp"]:
|
||||
official_title = titles["jp"]
|
||||
else:
|
||||
official_title = title_raw
|
||||
official_title, title_raw = TitleParser._resolve_titles(episode)
|
||||
if not title_raw:
|
||||
logger.warning("Cannot extract title_raw from '%s', skipping", raw)
|
||||
return None
|
||||
_season = episode.season
|
||||
if not official_title:
|
||||
logger.warning("Cannot extract official_title from '%s', skipping", raw)
|
||||
return None
|
||||
logger.debug("RAW:%s >> %s", raw, title_raw)
|
||||
return Bangumi(
|
||||
official_title=official_title,
|
||||
title_raw=title_raw,
|
||||
season=_season,
|
||||
season_raw=episode.season_raw,
|
||||
group_name=episode.group,
|
||||
dpi=episode.resolution,
|
||||
source=episode.source,
|
||||
subtitle=episode.sub,
|
||||
eps_collect=False if episode.episode > 1 else True,
|
||||
offset=0,
|
||||
filter=",".join(settings.rss_parser.filter),
|
||||
episode_type=episode.episode_type,
|
||||
)
|
||||
|
||||
# The upstream LLM path exposes its inferred non-episodic type on a
|
||||
# Bangumi. Keep that contract while routing deterministically parsed
|
||||
# movie markers into the dedicated local Movie table.
|
||||
if episode.episode_type == "movie" and not from_llm:
|
||||
return TitleParser._episode_to_movie(episode, official_title, title_raw)
|
||||
return TitleParser._episode_to_bangumi(episode, official_title, title_raw)
|
||||
except (ValueError, AttributeError, TypeError) as e:
|
||||
logger.warning(f"Cannot parse '{raw}': {type(e).__name__}: {e}")
|
||||
return None
|
||||
|
||||
@@ -2,7 +2,7 @@ import logging
|
||||
import re
|
||||
|
||||
from module.conf import settings
|
||||
from module.models import Bangumi, ResponseModel, RSSItem, Torrent
|
||||
from module.models import Bangumi, Movie, ResponseModel, RSSItem, Torrent
|
||||
from module.network import RequestContent
|
||||
from module.parser import TitleParser
|
||||
|
||||
@@ -12,6 +12,48 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RSSAnalyser:
|
||||
async def official_title_parser_movie(
|
||||
self,
|
||||
movie: Movie,
|
||||
rss: RSSItem,
|
||||
torrent: Torrent,
|
||||
fetch_poster: bool = True,
|
||||
):
|
||||
if not fetch_poster:
|
||||
pass
|
||||
elif rss.parser == "mikan":
|
||||
if not torrent.homepage:
|
||||
logger.warning("Mikan movie torrent has no homepage info.")
|
||||
else:
|
||||
try:
|
||||
poster_link, official_title = await TitleParser.mikan_parser(
|
||||
torrent.homepage
|
||||
)
|
||||
except AttributeError as e:
|
||||
logger.warning(
|
||||
f"Failed to parse Mikan homepage {torrent.homepage}: {e}"
|
||||
)
|
||||
else:
|
||||
movie.poster_link = poster_link
|
||||
if official_title:
|
||||
movie.official_title = official_title
|
||||
elif rss.parser == "tmdb":
|
||||
tmdb_title, _, year, poster_link = await TitleParser.tmdb_parser(
|
||||
movie.official_title,
|
||||
1,
|
||||
settings.rss_parser.language,
|
||||
episode_type="movie",
|
||||
)
|
||||
movie.official_title = tmdb_title
|
||||
if year:
|
||||
try:
|
||||
movie.year = int(year)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
movie.poster_link = poster_link
|
||||
if movie.official_title:
|
||||
movie.official_title = re.sub(r"[/:.\\]", " ", movie.official_title)
|
||||
|
||||
async def official_title_parser(
|
||||
self,
|
||||
bangumi: Bangumi,
|
||||
@@ -64,53 +106,86 @@ class RSSAnalyser:
|
||||
|
||||
async def torrents_to_data(
|
||||
self, torrents: list[Torrent], rss: RSSItem, full_parse: bool = True
|
||||
) -> list:
|
||||
new_data = []
|
||||
) -> tuple[list[Bangumi], list[Movie]]:
|
||||
new_bangumi: list[Bangumi] = []
|
||||
new_movies: list[Movie] = []
|
||||
seen_titles: set[str] = set()
|
||||
for torrent in torrents:
|
||||
bangumi = await TitleParser.raw_parser(raw=torrent.name)
|
||||
if bangumi and bangumi.title_raw not in seen_titles:
|
||||
await self.official_title_parser(
|
||||
bangumi=bangumi, rss=rss, torrent=torrent
|
||||
result = await TitleParser.raw_parser(raw=torrent.name)
|
||||
if result is None or not result.title_raw:
|
||||
continue
|
||||
title_raw = result.title_raw
|
||||
if title_raw in seen_titles:
|
||||
continue
|
||||
if isinstance(result, Movie):
|
||||
await self.official_title_parser_movie(
|
||||
movie=result, rss=rss, torrent=torrent
|
||||
)
|
||||
bangumi.rss_link = rss.url
|
||||
result.rss_link = rss.url
|
||||
seen_titles.add(title_raw)
|
||||
new_movies.append(result)
|
||||
logger.info(f"New movie found: {result.official_title}")
|
||||
elif isinstance(result, Bangumi):
|
||||
await self.official_title_parser(
|
||||
bangumi=result, rss=rss, torrent=torrent
|
||||
)
|
||||
result.rss_link = rss.url
|
||||
if not full_parse:
|
||||
return [bangumi]
|
||||
seen_titles.add(bangumi.title_raw)
|
||||
new_data.append(bangumi)
|
||||
logger.info(f"New bangumi founded: {bangumi.official_title}")
|
||||
return new_data
|
||||
return [result], new_movies
|
||||
seen_titles.add(title_raw)
|
||||
new_bangumi.append(result)
|
||||
logger.info(f"New bangumi found: {result.official_title}")
|
||||
return new_bangumi, new_movies
|
||||
|
||||
async def torrent_to_data(
|
||||
self, torrent: Torrent, rss: RSSItem, fetch_poster: bool = True
|
||||
) -> Bangumi | None:
|
||||
bangumi = await TitleParser.raw_parser(raw=torrent.name)
|
||||
if bangumi:
|
||||
await self.official_title_parser(
|
||||
bangumi=bangumi, rss=rss, torrent=torrent, fetch_poster=fetch_poster
|
||||
)
|
||||
bangumi.rss_link = rss.url
|
||||
return bangumi
|
||||
) -> Bangumi | Movie | None:
|
||||
result = await TitleParser.raw_parser(raw=torrent.name)
|
||||
if result:
|
||||
if isinstance(result, Movie):
|
||||
await self.official_title_parser_movie(
|
||||
movie=result,
|
||||
rss=rss,
|
||||
torrent=torrent,
|
||||
fetch_poster=fetch_poster,
|
||||
)
|
||||
result.rss_link = rss.url
|
||||
else:
|
||||
await self.official_title_parser(
|
||||
bangumi=result,
|
||||
rss=rss,
|
||||
torrent=torrent,
|
||||
fetch_poster=fetch_poster,
|
||||
)
|
||||
result.rss_link = rss.url
|
||||
return result
|
||||
return None
|
||||
|
||||
async def rss_to_data(
|
||||
self, rss: RSSItem, engine: RSSEngine, full_parse: bool = True
|
||||
) -> list[Bangumi]:
|
||||
rss_torrents = await self.get_rss_torrents(rss.url, full_parse)
|
||||
torrents_to_add = await engine.db.bangumi.match_list(rss_torrents, rss.url)
|
||||
# Filter out already-known movies first
|
||||
torrents_after_movies = await engine.db.movie.match_list(rss_torrents, rss.url)
|
||||
# Then filter out already-known bangumi
|
||||
torrents_to_add = await engine.db.bangumi.match_list(
|
||||
torrents_after_movies, rss.url
|
||||
)
|
||||
if not torrents_to_add:
|
||||
logger.debug("No new title has been found.")
|
||||
return []
|
||||
# New List
|
||||
new_data = await self.torrents_to_data(torrents_to_add, rss, full_parse)
|
||||
if new_data:
|
||||
# Add to database
|
||||
await engine.db.bangumi.add_all(new_data)
|
||||
return new_data
|
||||
else:
|
||||
return []
|
||||
# Parse remaining torrents
|
||||
new_bangumi, new_movies = await self.torrents_to_data(
|
||||
torrents_to_add, rss, full_parse
|
||||
)
|
||||
for movie in new_movies:
|
||||
await engine.db.movie.add(movie)
|
||||
if new_bangumi:
|
||||
await engine.db.bangumi.add_all(new_bangumi)
|
||||
return new_bangumi
|
||||
return []
|
||||
|
||||
async def link_to_data(self, rss: RSSItem) -> Bangumi | ResponseModel:
|
||||
async def link_to_data(self, rss: RSSItem) -> Bangumi | Movie | ResponseModel:
|
||||
torrents = await self.get_rss_torrents(rss.url, False)
|
||||
if not torrents:
|
||||
return ResponseModel(
|
||||
|
||||
@@ -10,7 +10,7 @@ from module.conf import settings
|
||||
from module.database import Database
|
||||
from module.database.bangumi import _groups_are_similar, match_bangumi_in_list
|
||||
from module.downloader import AddResult, DownloadClient
|
||||
from module.models import Bangumi, Episode, ResponseModel, RSSItem, Torrent
|
||||
from module.models import Bangumi, Episode, Movie, ResponseModel, RSSItem, Torrent
|
||||
from module.network import RequestContent
|
||||
from module.notification.events import (
|
||||
DownloadFailureEvent,
|
||||
@@ -383,6 +383,46 @@ class RSSEngine:
|
||||
await self.db.commit()
|
||||
return events
|
||||
|
||||
async def download_movie(self, movie: Movie):
|
||||
if not movie.rss_link:
|
||||
return ResponseModel(
|
||||
status=False,
|
||||
status_code=406,
|
||||
msg_en=f"Download movie {movie.official_title} failed: no RSS link.",
|
||||
msg_zh=f"下载剧场版 {movie.official_title} 失败:缺少 RSS 链接。",
|
||||
)
|
||||
async with RequestContent() as req:
|
||||
filter_pattern = movie.filter.replace(",", "|") if movie.filter else ""
|
||||
torrents = await req.get_torrents(movie.rss_link, filter_pattern)
|
||||
if torrents:
|
||||
async with DownloadClient() as client:
|
||||
result = await client.add_torrent(
|
||||
torrents, movie # type: ignore[arg-type]
|
||||
)
|
||||
if result is AddResult.FAILED:
|
||||
return ResponseModel(
|
||||
status=False,
|
||||
status_code=502,
|
||||
msg_en=f"Download movie {movie.official_title} failed.",
|
||||
msg_zh=f"下载剧场版 {movie.official_title} 失败。",
|
||||
)
|
||||
for torrent in torrents:
|
||||
torrent.downloaded = True
|
||||
await self.db.torrent.add_all(torrents)
|
||||
return ResponseModel(
|
||||
status=True,
|
||||
status_code=200,
|
||||
msg_en=f"Download movie {movie.official_title} successfully.",
|
||||
msg_zh=f"下载剧场版 {movie.official_title} 成功。",
|
||||
)
|
||||
else:
|
||||
return ResponseModel(
|
||||
status=False,
|
||||
status_code=406,
|
||||
msg_en=f"[Engine] Download movie {movie.official_title} failed.",
|
||||
msg_zh=f"下载剧场版 {movie.official_title} 失败。",
|
||||
)
|
||||
|
||||
async def download_bangumi(self, bangumi: Bangumi):
|
||||
async with RequestContent() as req:
|
||||
torrents = await req.get_torrents(
|
||||
|
||||
@@ -244,6 +244,60 @@ def _columns(engine, table: str) -> set[str]:
|
||||
return {c["name"] for c in inspect(engine).get_columns(table)}
|
||||
|
||||
|
||||
def _run_through_version(
|
||||
engine: Engine,
|
||||
version: int,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Run the real migration table only through one historical version."""
|
||||
historical = tuple(item for item in MIGRATIONS if item.version <= version)
|
||||
with monkeypatch.context() as patch:
|
||||
patch.setattr(migration_module, "MIGRATIONS", historical)
|
||||
patch.setattr(migration_module, "CURRENT_SCHEMA_VERSION", version)
|
||||
run_migrations(engine)
|
||||
|
||||
|
||||
def _make_v18_movie_engine(monkeypatch: pytest.MonkeyPatch) -> Engine:
|
||||
"""A database produced by the divergent tokenizer dev branch where v18=Movie."""
|
||||
engine = _make_v0_engine()
|
||||
_run_through_version(engine, 17, monkeypatch)
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("""CREATE TABLE movie (
|
||||
id INTEGER PRIMARY KEY,
|
||||
official_title VARCHAR NOT NULL,
|
||||
title_raw VARCHAR,
|
||||
year INTEGER,
|
||||
group_name VARCHAR,
|
||||
dpi VARCHAR,
|
||||
source VARCHAR,
|
||||
subtitle VARCHAR,
|
||||
poster_link VARCHAR,
|
||||
rss_link VARCHAR,
|
||||
added BOOLEAN NOT NULL DEFAULT 0,
|
||||
deleted BOOLEAN NOT NULL DEFAULT 0,
|
||||
save_path VARCHAR,
|
||||
rule_name VARCHAR,
|
||||
filter VARCHAR NOT NULL DEFAULT ''
|
||||
)"""))
|
||||
conn.execute(text("CREATE INDEX ix_movie_title_raw ON movie(title_raw)"))
|
||||
conn.execute(text("CREATE INDEX ix_movie_deleted ON movie(deleted)"))
|
||||
conn.execute(
|
||||
text(
|
||||
"INSERT INTO movie "
|
||||
"(id, official_title, title_raw, year, filter) VALUES "
|
||||
"(9, 'Preserved Movie', 'Preserved Movie', 2026, '')"
|
||||
)
|
||||
)
|
||||
conn.execute(
|
||||
text(
|
||||
"INSERT INTO user (id, username, password) VALUES "
|
||||
"(7, 'movie_dev_user', 'hashed-password')"
|
||||
)
|
||||
)
|
||||
set_schema_version(conn, 18)
|
||||
return engine
|
||||
|
||||
|
||||
class TestMigrationTable:
|
||||
def test_every_migration_has_an_already_applied_guard(self):
|
||||
"""No migration may rely on the old hardcoded if-ladder."""
|
||||
@@ -277,6 +331,7 @@ class TestRunMigrations:
|
||||
assert {"enabled", "created_at", "updated_at"} <= _columns(engine, "user")
|
||||
assert "auth_session" in inspector.get_table_names()
|
||||
assert "api_token" in inspector.get_table_names()
|
||||
assert "movie" in inspector.get_table_names()
|
||||
user_indexes = {ix["name"] for ix in inspector.get_indexes("user")}
|
||||
assert "ix_user_username" in user_indexes
|
||||
|
||||
@@ -344,6 +399,59 @@ class TestRunMigrations:
|
||||
indexes = {ix["name"]: ix for ix in inspector.get_indexes("llmcredential")}
|
||||
assert indexes["ix_llmcredential_provider_id"]["unique"]
|
||||
|
||||
def test_upgrades_auth_beta_v20_without_losing_tokens(self, monkeypatch):
|
||||
engine = _make_v19_auth_engine()
|
||||
_run_through_version(engine, 20, monkeypatch)
|
||||
with engine.connect() as conn:
|
||||
before = conn.execute(
|
||||
text("SELECT id, token_hash, scope, prefix FROM api_token ORDER BY id")
|
||||
).all()
|
||||
|
||||
run_migrations(engine)
|
||||
|
||||
inspector = inspect(engine)
|
||||
assert "movie" in inspector.get_table_names()
|
||||
assert {"enabled", "created_at", "updated_at"} <= _columns(engine, "user")
|
||||
with engine.connect() as conn:
|
||||
after = conn.execute(
|
||||
text("SELECT id, token_hash, scope, prefix FROM api_token ORDER BY id")
|
||||
).all()
|
||||
assert get_schema_version(conn) == CURRENT_SCHEMA_VERSION
|
||||
assert after == before
|
||||
|
||||
def test_repairs_divergent_movie_v18_database(self, monkeypatch):
|
||||
engine = _make_v18_movie_engine(monkeypatch)
|
||||
|
||||
run_migrations(engine)
|
||||
|
||||
inspector = inspect(engine)
|
||||
assert {"movie", "auth_session", "api_token"} <= set(
|
||||
inspector.get_table_names()
|
||||
)
|
||||
assert {"enabled", "created_at", "updated_at"} <= _columns(engine, "user")
|
||||
user_indexes = {index["name"] for index in inspector.get_indexes("user")}
|
||||
assert {"ix_user_username", "ix_user_enabled"} <= user_indexes
|
||||
token_indexes = {index["name"] for index in inspector.get_indexes("api_token")}
|
||||
assert "ix_api_token_token_hash_scope" in token_indexes
|
||||
with engine.connect() as conn:
|
||||
assert (
|
||||
conn.execute(
|
||||
text("SELECT official_title FROM movie WHERE id = 9")
|
||||
).scalar_one()
|
||||
== "Preserved Movie"
|
||||
)
|
||||
user = conn.execute(
|
||||
text(
|
||||
"SELECT username, enabled, created_at, updated_at "
|
||||
"FROM user WHERE id = 7"
|
||||
)
|
||||
).one()
|
||||
assert user[0] == "movie_dev_user"
|
||||
assert user[1] == 1
|
||||
assert user[2] is not None
|
||||
assert user[3] is not None
|
||||
assert get_schema_version(conn) == CURRENT_SCHEMA_VERSION
|
||||
|
||||
def test_v20_rebuilds_v19_api_tokens_without_losing_data(self):
|
||||
engine = _make_v19_auth_engine()
|
||||
# Production upgrade paths may call metadata.create_all before the
|
||||
@@ -378,7 +486,7 @@ class TestRunMigrations:
|
||||
.mappings()
|
||||
.all()
|
||||
)
|
||||
assert get_schema_version(conn) == 20
|
||||
assert get_schema_version(conn) == CURRENT_SCHEMA_VERSION
|
||||
|
||||
assert len(after) == len(before) == 2
|
||||
for old, new in zip(before, after):
|
||||
@@ -420,7 +528,7 @@ class TestRunMigrations:
|
||||
|
||||
def test_v20_rebuild_rolls_back_if_a_late_statement_fails(self, monkeypatch):
|
||||
engine = _make_v19_auth_engine()
|
||||
v20 = MIGRATIONS[-1]
|
||||
v20 = next(item for item in MIGRATIONS if item.version == 20)
|
||||
broken_v20 = replace(
|
||||
v20,
|
||||
statements=(
|
||||
@@ -432,7 +540,7 @@ class TestRunMigrations:
|
||||
monkeypatch.setattr(
|
||||
migration_module,
|
||||
"MIGRATIONS",
|
||||
(*MIGRATIONS[:-1], broken_v20),
|
||||
tuple(broken_v20 if item.version == 20 else item for item in MIGRATIONS),
|
||||
)
|
||||
|
||||
with pytest.raises(OperationalError):
|
||||
@@ -481,7 +589,7 @@ class TestRunMigrations:
|
||||
prefix = conn.execute(
|
||||
text("SELECT prefix FROM api_token WHERE id = 1")
|
||||
).scalar_one()
|
||||
assert get_schema_version(conn) == 20
|
||||
assert get_schema_version(conn) == CURRENT_SCHEMA_VERSION
|
||||
assert prefix == "legacy_cccccccc"
|
||||
|
||||
def test_is_idempotent(self):
|
||||
|
||||
81
backend/src/test/test_movie_parser.py
Normal file
81
backend/src/test/test_movie_parser.py
Normal file
@@ -0,0 +1,81 @@
|
||||
import pytest
|
||||
|
||||
from module.parser.analyser import raw_parser
|
||||
|
||||
|
||||
class TestMovieMarkerDetection:
|
||||
def test_gekijouban_chinese_traditional(self):
|
||||
content = "[SubGroup] 劇場版 Some Anime Title [1080p]"
|
||||
info = raw_parser(content)
|
||||
assert info is not None
|
||||
assert info.is_movie is True
|
||||
|
||||
def test_gekijouban_chinese_simplified(self):
|
||||
content = "[SubGroup] 剧场版 Some Anime Title [1080p]"
|
||||
info = raw_parser(content)
|
||||
assert info is not None
|
||||
assert info.is_movie is True
|
||||
|
||||
def test_movie_keyword_english(self):
|
||||
content = "[SubGroup] Some Anime Title Movie [1080p HEVC]"
|
||||
info = raw_parser(content)
|
||||
assert info is not None
|
||||
assert info.is_movie is True
|
||||
|
||||
def test_gekijouban_romaji(self):
|
||||
content = "[SubGroup] Gekijouban Some Anime [1080p]"
|
||||
info = raw_parser(content)
|
||||
assert info is not None
|
||||
assert info.is_movie is True
|
||||
|
||||
def test_movie_keyword_case_insensitive(self):
|
||||
content = "[SubGroup] Some Anime MOVIE [720p]"
|
||||
info = raw_parser(content)
|
||||
assert info is not None
|
||||
assert info.is_movie is True
|
||||
|
||||
def test_gekijouban_with_hyphen(self):
|
||||
content = "[SubGroup] Gekijou-ban Title [1080p]"
|
||||
info = raw_parser(content)
|
||||
assert info is not None
|
||||
assert info.is_movie is True
|
||||
|
||||
def test_movie_defaults(self):
|
||||
"""Movies should default to episode=0, season=1."""
|
||||
content = "[SubGroup] 劇場版 Anime Title [1080p]"
|
||||
info = raw_parser(content)
|
||||
assert info is not None
|
||||
assert info.is_movie is True
|
||||
assert info.episode == 0
|
||||
assert info.season == 1
|
||||
|
||||
|
||||
class TestNonMovieRegression:
|
||||
def test_regular_episode_not_movie(self):
|
||||
content = "[LoliHouse] Some Anime - 01 [WebRip 1080p HEVC-10bit AAC]"
|
||||
info = raw_parser(content)
|
||||
assert info is not None
|
||||
assert info.is_movie is False
|
||||
assert info.episode == 1
|
||||
|
||||
def test_regular_series_with_season(self):
|
||||
content = "【幻樱字幕组】【Anime Title S02】【12】【GB_MP4】【1920X1080】"
|
||||
info = raw_parser(content)
|
||||
assert info is not None
|
||||
assert info.is_movie is False
|
||||
assert info.season == 2
|
||||
assert info.episode == 12
|
||||
|
||||
def test_regular_series_chinese_episode(self):
|
||||
content = "[字幕组] 动画标题 第5话 [1080p]"
|
||||
info = raw_parser(content)
|
||||
assert info is not None
|
||||
assert info.is_movie is False
|
||||
assert info.episode == 5
|
||||
|
||||
def test_end_marker_not_movie(self):
|
||||
content = "[LoliHouse] Anime Title - 12 [WebRip 1080p HEVC-10bit AAC][END]"
|
||||
info = raw_parser(content)
|
||||
assert info is not None
|
||||
assert info.is_movie is False
|
||||
assert info.episode == 12
|
||||
Reference in New Issue
Block a user