From 9d5b1c399875610170256df8a379a0e44c084a0d Mon Sep 17 00:00:00 2001 From: Estrella Pan Date: Mon, 13 Jul 2026 01:36:17 +0200 Subject: [PATCH] fix(renamer): safely replace higher revisions (#1078) --- backend/src/module/api/downloader.py | 38 +- backend/src/module/conf/const.py | 5 + backend/src/module/core/loops.py | 3 + backend/src/module/database/aria2.py | 107 ++ backend/src/module/database/combine.py | 2 + backend/src/module/database/migrations.py | 166 +- .../src/module/database/rename_operation.py | 419 +++++ backend/src/module/downloader/__init__.py | 4 + backend/src/module/downloader/base.py | 36 +- .../downloader/client/aria2_downloader.py | 248 ++- .../downloader/client/mock_downloader.py | 14 +- .../module/downloader/client/qb_downloader.py | 52 +- .../src/module/downloader/download_client.py | 45 +- backend/src/module/manager/renamer.py | 1370 +++++++++++++++-- backend/src/module/manager/revision_policy.py | 105 ++ backend/src/module/models/__init__.py | 5 + backend/src/module/models/aria2.py | 1 + backend/src/module/models/config.py | 4 + backend/src/module/models/rename_operation.py | 122 ++ backend/src/module/notification/__init__.py | 2 + backend/src/module/notification/events.py | 33 + backend/src/test/conftest.py | 5 +- backend/src/test/test_api_downloader.py | 71 + backend/src/test/test_aria2_downloader.py | 310 +++- backend/src/test/test_config.py | 19 + .../test/test_database_rename_operation.py | 270 ++++ backend/src/test/test_download_client.py | 32 +- backend/src/test/test_downloader_protocol.py | 12 + backend/src/test/test_loops.py | 27 + backend/src/test/test_migrations_module.py | 112 ++ backend/src/test/test_mock_downloader.py | 12 +- backend/src/test/test_qb_downloader.py | 62 +- backend/src/test/test_renamer.py | 517 ++++++- backend/src/test/test_revision_policy.py | 58 + ...-issue-1078-revision-replacement-design.md | 18 +- .../layout/ab-notification-center.vue | 1 + .../setting/__tests__/config-manage.test.ts | 83 + .../src/components/setting/config-manage.vue | 35 +- webui/src/i18n/en.json | 8 + webui/src/i18n/zh-CN.json | 8 + webui/src/pages/index/config.vue | 19 +- webui/src/store/notification.ts | 1 + webui/src/test/mocks/api.ts | 1 + webui/types/config.ts | 4 + 44 files changed, 4276 insertions(+), 190 deletions(-) create mode 100644 backend/src/module/database/rename_operation.py create mode 100644 backend/src/module/manager/revision_policy.py create mode 100644 backend/src/module/models/rename_operation.py create mode 100644 backend/src/test/test_database_rename_operation.py create mode 100644 backend/src/test/test_revision_policy.py create mode 100644 webui/src/components/setting/__tests__/config-manage.test.ts diff --git a/backend/src/module/api/downloader.py b/backend/src/module/api/downloader.py index 6de81a96..040b9936 100644 --- a/backend/src/module/api/downloader.py +++ b/backend/src/module/api/downloader.py @@ -1,7 +1,7 @@ import logging import re -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel from module.database import Database @@ -40,6 +40,42 @@ async def get_torrents(): return await client.get_torrent_info(category="Bangumi", status_filter=None) +@router.get("/rename-conflicts", dependencies=[Depends(get_current_user)]) +async def get_rename_conflicts(): + """List durable media rename conflicts awaiting user action.""" + + async with Database() as db: + rows = await db.rename_operation.list_conflicts() + return [row.model_dump(mode="json") for row in rows] + + +@router.post( + "/rename-conflicts/{operation_id}/retry", + dependencies=[Depends(get_current_user)], +) +async def retry_rename_conflict(operation_id: int): + """Clear one terminal conflict so the next rename pass revalidates it.""" + + async with Database() as db: + row = await db.rename_operation.get(operation_id) + if row is None: + raise HTTPException(status_code=404, detail="Rename conflict not found") + if row.state != "conflict" or row.kind != "conflict": + raise HTTPException( + status_code=409, + detail=( + "Only non-destructive terminal conflicts can be retried; " + "replacement recovery state must be preserved" + ), + ) + await db.rename_operation.delete(operation_id) + return { + "status": True, + "msg_en": "Rename conflict cleared; it will be revalidated", + "msg_zh": "重命名冲突已清除，将在下一轮重新校验", + } + + @router.post("/torrents/pause", dependencies=[Depends(get_current_user)]) async def pause_torrents(req: TorrentHashesRequest): hashes = "|".join(req.hashes) diff --git a/backend/src/module/conf/const.py b/backend/src/module/conf/const.py index 37557509..90a235fb 100644 --- a/backend/src/module/conf/const.py +++ b/backend/src/module/conf/const.py @@ -31,6 +31,7 @@ DEFAULT_SETTINGS: dict[str, dict[str, Any]] = { "rename_method": "pn", "group_tag": False, "remove_bad_torrent": False, + "revision_conflict_policy": "hold", }, "log": { "debug_enable": False, @@ -112,6 +113,10 @@ ENV_TO_ATTR: dict[str, dict[str, Any]] = { "remove_bad_torrent", lambda e: e.lower() in ("true", "1", "t"), ), + "AB_REVISION_CONFLICT_POLICY": ( + "revision_conflict_policy", + lambda e: e.lower(), + ), }, "log": { "AB_DEBUG_MODE": ("debug_enable", lambda e: e.lower() in ("true", "1", "t")), diff --git a/backend/src/module/core/loops.py b/backend/src/module/core/loops.py index c641c367..87062c02 100644 --- a/backend/src/module/core/loops.py +++ b/backend/src/module/core/loops.py @@ -65,6 +65,9 @@ async def rename_tick(notifier: NotificationManager) -> None: async with DownloadClient() as client: renamer = Renamer(client) renamed_info = await renamer.rename() + rename_events = list(renamer.events) + if rename_events: + await asyncio.gather(*[notifier.send_event(event) for event in rename_events]) if settings.notification.enable and renamed_info: # Gather across renamed items so a batch rename (e.g. first import of # a season) doesn't serialize N notification round-trips. diff --git a/backend/src/module/database/aria2.py b/backend/src/module/database/aria2.py index 469b5ea3..139ef030 100644 --- a/backend/src/module/database/aria2.py +++ b/backend/src/module/database/aria2.py @@ -6,6 +6,7 @@ import json import logging +from dataclasses import asdict, dataclass from sqlalchemy.ext.asyncio import AsyncSession from sqlmodel import select @@ -15,6 +16,41 @@ from module.models.aria2 import Aria2Gid logger = logging.getLogger(__name__) +@dataclass(frozen=True) +class Aria2RenameIntent: + """Durable proof that this gid owned a filesystem move before it started.""" + + old_path: str + new_path: str + st_dev: int + st_ino: int + st_size: int + st_mtime_ns: int + + def to_json(self) -> str: + return json.dumps( + {"version": 1, **asdict(self)}, ensure_ascii=False, sort_keys=True + ) + + @classmethod + def from_json(cls, raw: str | None) -> "Aria2RenameIntent | None": + if not raw: + return None + try: + data = json.loads(raw) + except (json.JSONDecodeError, TypeError): + return None + if not isinstance(data, dict) or data.get("version") != 1: + return None + path_fields = ("old_path", "new_path") + stat_fields = ("st_dev", "st_ino", "st_size", "st_mtime_ns") + if any(not isinstance(data.get(field), str) for field in path_fields): + return None + if any(type(data.get(field)) is not int for field in stat_fields): + return None + return cls(**{field: data[field] for field in (*path_fields, *stat_fields)}) + + class Aria2GidDatabase: def __init__(self, session: AsyncSession): self.session = session @@ -88,6 +124,7 @@ class Aria2GidDatabase: category=old.category, dedup_key=old.dedup_key, renamed_paths=old.renamed_paths, + rename_intent=old.rename_intent, created_at=old.created_at, ) ) @@ -101,6 +138,8 @@ class Aria2GidDatabase: existing.renamed_paths = _merge_renamed_paths( existing.renamed_paths, old.renamed_paths ) + if existing.rename_intent is None: + existing.rename_intent = old.rename_intent self.session.add(existing) await self.session.delete(old) await self.session.commit() @@ -126,6 +165,62 @@ class Aria2GidDatabase: mapping[old_path] = new_path await self.upsert(gid, renamed_paths=json.dumps(mapping, ensure_ascii=False)) + async def get_rename_intent(self, gid: str) -> Aria2RenameIntent | None: + record = await self.session.get(Aria2Gid, gid) + if record is None or not record.rename_intent: + return None + intent = Aria2RenameIntent.from_json(record.rename_intent) + if intent is None: + logger.warning("Ignoring invalid rename_intent for gid %s", gid) + return intent + + async def set_rename_intent(self, gid: str, intent: Aria2RenameIntent) -> None: + record = await self.session.get(Aria2Gid, gid) + if record is None: + record = Aria2Gid(gid=gid) + record.rename_intent = intent.to_json() + self.session.add(record) + await self.session.commit() + + async def clear_rename_intent( + self, + gid: str, + expected: Aria2RenameIntent | None = None, + ) -> bool: + record = await self.session.get(Aria2Gid, gid) + if record is None: + return False + if ( + expected is not None + and Aria2RenameIntent.from_json(record.rename_intent) != expected + ): + return False + record.rename_intent = None + self.session.add(record) + await self.session.commit() + return True + + async def finalize_rename_intent( + self, gid: str, expected: Aria2RenameIntent + ) -> bool: + """Commit the sidecar mapping and clear its matching intent together.""" + record = await self.session.get(Aria2Gid, gid) + if ( + record is None + or Aria2RenameIntent.from_json(record.rename_intent) != expected + ): + return False + mapping = _decode_renamed_paths(record.renamed_paths) + for original_path, renamed_path in list(mapping.items()): + if renamed_path == expected.old_path: + mapping[original_path] = expected.new_path + mapping[expected.old_path] = expected.new_path + record.renamed_paths = json.dumps(mapping, ensure_ascii=False) + record.rename_intent = None + self.session.add(record) + await self.session.commit() + return True + async def delete(self, gid: str) -> None: existing = await self.session.get(Aria2Gid, gid) if existing is not None: @@ -147,3 +242,15 @@ def _merge_renamed_paths(left: str | None, right: str | None) -> str | None: return left merged.update(incoming) return json.dumps(merged, ensure_ascii=False) + + +def _decode_renamed_paths(raw: str | None) -> dict[str, str]: + if not raw: + return {} + try: + data = json.loads(raw) + except json.JSONDecodeError: + return {} + if not isinstance(data, dict): + return {} + return {str(key): str(value) for key, value in data.items()} diff --git a/backend/src/module/database/combine.py b/backend/src/module/database/combine.py index ee15dada..72beef48 100644 --- a/backend/src/module/database/combine.py +++ b/backend/src/module/database/combine.py @@ -20,6 +20,7 @@ from .migrations import ( # noqa: F401 (re-exported for existing importers) run_migrations_async, ) from .movie import MovieDatabase +from .rename_operation import RenameOperationDatabase from .rss import RSSDatabase from .torrent import TorrentDatabase from .user import UserDatabase @@ -51,6 +52,7 @@ class Database: self.auth = AuthDatabase(self.session) self.inbox = InboxDatabase(self.session) self.llm_credential = LLMCredentialDatabase(self.session) + self.rename_operation = RenameOperationDatabase(self.session) async def __aenter__(self): return self diff --git a/backend/src/module/database/migrations.py b/backend/src/module/database/migrations.py index 7e482c01..7aac816f 100644 --- a/backend/src/module/database/migrations.py +++ b/backend/src/module/database/migrations.py @@ -17,7 +17,7 @@ from pydantic_core import PydanticUndefined from sqlalchemy import Connection, Engine, inspect, text from sqlmodel import SQLModel -from module.models import ApiToken, AuthSession, Bangumi, Movie, User +from module.models import ApiToken, AuthSession, Bangumi, Movie, RenameOperation, User from module.models.inbox import InboxMessage from module.models.llm_credential import LLMCredential from module.models.passkey import Passkey @@ -38,6 +38,7 @@ TABLE_MODELS: list[type[SQLModel]] = [ LLMCredential, AuthSession, ApiToken, + RenameOperation, ] # already_applied 守卫：接收 inspector，返回该迁移是否已生效 @@ -617,6 +618,169 @@ MIGRATIONS: tuple[Migration, ...] = ( ), ), ), + Migration( + 23, + "create durable rename/revision-replacement operation state", + ( + """CREATE TABLE IF NOT EXISTS rename_operation ( + id INTEGER PRIMARY KEY, + downloader_type VARCHAR(32) NOT NULL, + kind VARCHAR(32) NOT NULL DEFAULT 'conflict', + state VARCHAR(32) NOT NULL DEFAULT 'planned', + new_task_id VARCHAR NOT NULL, + old_task_id VARCHAR, + save_path VARCHAR NOT NULL, + source_path VARCHAR NOT NULL, + target_path VARCHAR NOT NULL, + staged_path VARCHAR, + bangumi_id INTEGER, + media_type VARCHAR(32), + season INTEGER, + episode REAL, + group_name VARCHAR, + resolution VARCHAR(32), + old_revision INTEGER, + new_revision INTEGER, + revision_metadata TEXT, + attempt_count INTEGER NOT NULL DEFAULT 0, + retry_at TIMESTAMP, + lease_owner VARCHAR(64), + lease_expires_at TIMESTAMP, + notified_at TIMESTAMP, + last_error TEXT, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT ck_rename_operation_state CHECK ( + state IN ('conflict', 'retry', 'running', 'planned', 'old_staged', + 'new_promoted', 'old_removed', 'done') + ), + CONSTRAINT ck_rename_operation_attempt_count CHECK ( + attempt_count >= 0 + ) + )""", + "CREATE UNIQUE INDEX IF NOT EXISTS ux_rename_operation_identity " + "ON rename_operation(downloader_type, new_task_id, save_path, " + "source_path, target_path)", + "CREATE UNIQUE INDEX IF NOT EXISTS ux_rename_operation_active_target " + "ON rename_operation(downloader_type, save_path, target_path) " + "WHERE state NOT IN ('done')", + "CREATE INDEX IF NOT EXISTS ix_rename_operation_state_retry_at " + "ON rename_operation(state, retry_at)", + "CREATE INDEX IF NOT EXISTS ix_rename_operation_new_task_id " + "ON rename_operation(new_task_id)", + "CREATE INDEX IF NOT EXISTS ix_rename_operation_old_task_id " + "ON rename_operation(old_task_id)", + ), + all_checks( + table_exists("rename_operation"), + index_exists("rename_operation", "ux_rename_operation_identity"), + index_exists("rename_operation", "ux_rename_operation_active_target"), + index_exists("rename_operation", "ix_rename_operation_state_retry_at"), + index_exists("rename_operation", "ix_rename_operation_new_task_id"), + index_exists("rename_operation", "ix_rename_operation_old_task_id"), + ), + ( + ( + """CREATE TABLE IF NOT EXISTS rename_operation ( + id INTEGER PRIMARY KEY, + downloader_type VARCHAR(32) NOT NULL, + kind VARCHAR(32) NOT NULL DEFAULT 'conflict', + state VARCHAR(32) NOT NULL DEFAULT 'planned', + new_task_id VARCHAR NOT NULL, + old_task_id VARCHAR, + save_path VARCHAR NOT NULL, + source_path VARCHAR NOT NULL, + target_path VARCHAR NOT NULL, + staged_path VARCHAR, + bangumi_id INTEGER, + media_type VARCHAR(32), + season INTEGER, + episode REAL, + group_name VARCHAR, + resolution VARCHAR(32), + old_revision INTEGER, + new_revision INTEGER, + revision_metadata TEXT, + attempt_count INTEGER NOT NULL DEFAULT 0, + retry_at TIMESTAMP, + lease_owner VARCHAR(64), + lease_expires_at TIMESTAMP, + notified_at TIMESTAMP, + last_error TEXT, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT ck_rename_operation_state CHECK ( + state IN ('conflict', 'retry', 'running', 'planned', 'old_staged', + 'new_promoted', 'old_removed', 'done') + ), + CONSTRAINT ck_rename_operation_attempt_count CHECK ( + attempt_count >= 0 + ) + )""", + table_exists("rename_operation"), + ), + ( + "CREATE UNIQUE INDEX IF NOT EXISTS ux_rename_operation_identity " + "ON rename_operation(downloader_type, new_task_id, save_path, " + "source_path, target_path)", + index_exists("rename_operation", "ux_rename_operation_identity"), + ), + ( + "CREATE UNIQUE INDEX IF NOT EXISTS ux_rename_operation_active_target " + "ON rename_operation(downloader_type, save_path, target_path) " + "WHERE state NOT IN ('done')", + index_exists("rename_operation", "ux_rename_operation_active_target"), + ), + ( + "CREATE INDEX IF NOT EXISTS ix_rename_operation_state_retry_at " + "ON rename_operation(state, retry_at)", + index_exists("rename_operation", "ix_rename_operation_state_retry_at"), + ), + ( + "CREATE INDEX IF NOT EXISTS ix_rename_operation_new_task_id " + "ON rename_operation(new_task_id)", + index_exists("rename_operation", "ix_rename_operation_new_task_id"), + ), + ( + "CREATE INDEX IF NOT EXISTS ix_rename_operation_old_task_id " + "ON rename_operation(old_task_id)", + index_exists("rename_operation", "ix_rename_operation_old_task_id"), + ), + ), + ), + Migration( + 24, + "add durable filesystem rename intent to aria2 gid state", + ( + """CREATE TABLE IF NOT EXISTS aria2_gid ( + gid VARCHAR NOT NULL PRIMARY KEY, + bangumi_id INTEGER REFERENCES bangumi(id), + category VARCHAR, + dedup_key VARCHAR, + renamed_paths TEXT DEFAULT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP + )""", + "ALTER TABLE aria2_gid ADD COLUMN rename_intent TEXT DEFAULT NULL", + ), + column_exists("aria2_gid", "rename_intent"), + ( + ( + """CREATE TABLE IF NOT EXISTS aria2_gid ( + gid VARCHAR NOT NULL PRIMARY KEY, + bangumi_id INTEGER REFERENCES bangumi(id), + category VARCHAR, + dedup_key VARCHAR, + renamed_paths TEXT DEFAULT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP + )""", + table_exists("aria2_gid"), + ), + ( + "ALTER TABLE aria2_gid ADD COLUMN rename_intent TEXT DEFAULT NULL", + column_exists("aria2_gid", "rename_intent"), + ), + ), + ), ) # 由迁移列表派生，新增迁移时无需手动同步 diff --git a/backend/src/module/database/rename_operation.py b/backend/src/module/database/rename_operation.py new file mode 100644 index 00000000..a9d60cec --- /dev/null +++ b/backend/src/module/database/rename_operation.py @@ -0,0 +1,419 @@ +"""Repository for durable rename/revision-replacement operations (#1078).""" + +from datetime import datetime, timedelta + +from sqlalchemy import delete, or_, update +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession +from sqlmodel import col, select + +from module.models.rename_operation import ( + RENAME_OPERATION_STATES, + RenameOperation, + RenameOperationState, + utc_now, +) + + +def _validate_state(state: str) -> None: + if state not in RENAME_OPERATION_STATES: + raise ValueError(f"Unsupported rename operation state: {state}") + + +class RenameOperationDatabase: + def __init__(self, session: AsyncSession): + self.session = session + + async def get(self, operation_id: int | None) -> RenameOperation | None: + if operation_id is None: + return None + return await self.session.get(RenameOperation, operation_id) + + async def get_by_identity( + self, + *, + downloader_type: str, + new_task_id: str, + save_path: str, + source_path: str, + target_path: str, + ) -> RenameOperation | None: + result = await self.session.execute( + select(RenameOperation).where( + RenameOperation.downloader_type == downloader_type, + RenameOperation.new_task_id == new_task_id, + RenameOperation.save_path == save_path, + RenameOperation.source_path == source_path, + RenameOperation.target_path == target_path, + ) + ) + return result.scalar_one_or_none() + + async def get_by_target( + self, + *, + downloader_type: str, + save_path: str, + target_path: str, + active_only: bool = True, + ) -> RenameOperation | None: + statement = select(RenameOperation).where( + RenameOperation.downloader_type == downloader_type, + RenameOperation.save_path == save_path, + RenameOperation.target_path == target_path, + ) + if active_only: + statement = statement.where(RenameOperation.state != "done") + statement = statement.order_by( # type: ignore[assignment] + col(RenameOperation.updated_at).desc() + ) + result = await self.session.execute(statement) + return result.scalars().first() + + async def get_or_create( + self, operation: RenameOperation + ) -> tuple[RenameOperation, bool]: + """Return the identity row, creating it when absent. + + The pre-read handles the normal idempotent path. The database unique + indexes remain authoritative for concurrent writers; an identity race + is re-read after rollback, while an active-target collision is exposed + as ``IntegrityError`` for the caller to reconcile. + """ + + existing = await self.get_by_identity( + downloader_type=operation.downloader_type, + new_task_id=operation.new_task_id, + save_path=operation.save_path, + source_path=operation.source_path, + target_path=operation.target_path, + ) + if existing is not None: + return existing, False + + operation.created_at = operation.created_at or utc_now() + operation.updated_at = utc_now() + try: + # Keep an active-target race inside a SAVEPOINT. Rolling back the + # whole session would expire unrelated rows already returned to the + # caller and make a recoverable uniqueness conflict poison the + # surrounding operation. + async with self.session.begin_nested(): + self.session.add(operation) + await self.session.flush() + except IntegrityError: + existing = await self.get_by_identity( + downloader_type=operation.downloader_type, + new_task_id=operation.new_task_id, + save_path=operation.save_path, + source_path=operation.source_path, + target_path=operation.target_path, + ) + if existing is not None: + return existing, False + raise + await self.session.commit() + await self.session.refresh(operation) + return operation, True + + async def upsert_conflict( + self, operation: RenameOperation + ) -> tuple[RenameOperation, bool]: + """Persist a terminal hold conflict without duplicating notifications.""" + + operation.state = "conflict" + operation.retry_at = None + existing = await self.get_by_identity( + downloader_type=operation.downloader_type, + new_task_id=operation.new_task_id, + save_path=operation.save_path, + source_path=operation.source_path, + target_path=operation.target_path, + ) + if existing is None: + return await self.get_or_create(operation) + + # Refresh reconciliation metadata without resetting attempts, + # notification history, or the original creation timestamp. + for field_name in ( + "kind", + "old_task_id", + "staged_path", + "bangumi_id", + "media_type", + "season", + "episode", + "group_name", + "resolution", + "old_revision", + "new_revision", + "revision_metadata", + "last_error", + ): + setattr(existing, field_name, getattr(operation, field_name)) + existing.state = "conflict" + existing.retry_at = None + existing.updated_at = utc_now() + self.session.add(existing) + await self.session.commit() + await self.session.refresh(existing) + return existing, False + + async def set_state( + self, + operation_id: int | None, + state: RenameOperationState, + *, + retry_at: datetime | None = None, + last_error: str | None = None, + ) -> RenameOperation | None: + _validate_state(state) + row = await self.get(operation_id) + if row is None: + return None + row.state = state + row.retry_at = retry_at + row.last_error = last_error + row.updated_at = utc_now() + self.session.add(row) + await self.session.commit() + await self.session.refresh(row) + return row + + async def claim( + self, + operation_id: int | None, + *, + from_states: tuple[RenameOperationState, ...], + to_state: RenameOperationState, + now: datetime | None = None, + ) -> RenameOperation | None: + """Atomically claim a due operation and count one execution attempt.""" + + if operation_id is None or not from_states: + return None + for state in from_states: + _validate_state(state) + _validate_state(to_state) + claimed_at = now or utc_now() + result = await self.session.execute( + update(RenameOperation) + .where( + col(RenameOperation.id) == operation_id, + col(RenameOperation.state).in_(from_states), + or_( + col(RenameOperation.retry_at).is_(None), + col(RenameOperation.retry_at) <= claimed_at, + ), + ) + .values( + state=to_state, + attempt_count=RenameOperation.attempt_count + 1, + retry_at=None, + updated_at=claimed_at, + ) + .execution_options(synchronize_session="fetch") + ) + await self.session.commit() + if not result.rowcount: # type: ignore[attr-defined] + return None + row = await self.get(operation_id) + if row is not None: + await self.session.refresh(row) + return row + + async def recover_stale_running( + self, operation_id: int | None, *, before: datetime + ) -> bool: + """CAS a crashed ordinary rename lease back to retryable state.""" + + if operation_id is None: + return False + now = utc_now() + result = await self.session.execute( + update(RenameOperation) + .where( + col(RenameOperation.id) == operation_id, + col(RenameOperation.state) == "running", + col(RenameOperation.updated_at) <= before, + ) + .values(state="retry", retry_at=None, updated_at=now) + .execution_options(synchronize_session="fetch") + ) + await self.session.commit() + return bool(result.rowcount) # type: ignore[attr-defined] + + async def claim_replacement_lease( + self, + operation_id: int | None, + *, + owner: str, + now: datetime | None = None, + lease_for: timedelta = timedelta(minutes=5), + ) -> RenameOperation | None: + """Fence one replacement Saga across processes for a bounded lease.""" + + if operation_id is None: + return None + claimed_at = now or utc_now() + result = await self.session.execute( + update(RenameOperation) + .where( + col(RenameOperation.id) == operation_id, + col(RenameOperation.kind) == "replacement", + col(RenameOperation.state).in_( + ("planned", "old_staged", "new_promoted", "old_removed") + ), + or_( + col(RenameOperation.lease_owner).is_(None), + col(RenameOperation.lease_expires_at).is_(None), + col(RenameOperation.lease_expires_at) <= claimed_at, + ), + ) + .values( + lease_owner=owner, + lease_expires_at=claimed_at + lease_for, + attempt_count=RenameOperation.attempt_count + 1, + updated_at=claimed_at, + ) + .execution_options(synchronize_session="fetch") + ) + await self.session.commit() + if not result.rowcount: # type: ignore[attr-defined] + return None + return await self.get(operation_id) + + async def set_state_claimed( + self, + operation_id: int | None, + *, + owner: str, + state: RenameOperationState, + retry_at: datetime | None = None, + last_error: str | None = None, + ) -> RenameOperation | None: + """Advance a replacement only while the caller still owns its lease.""" + + if operation_id is None: + return None + _validate_state(state) + now = utc_now() + result = await self.session.execute( + update(RenameOperation) + .where( + col(RenameOperation.id) == operation_id, + col(RenameOperation.lease_owner) == owner, + col(RenameOperation.lease_expires_at) > now, + ) + .values( + state=state, + retry_at=retry_at, + last_error=last_error, + lease_owner=None, + lease_expires_at=None, + updated_at=now, + ) + .execution_options(synchronize_session="fetch") + ) + await self.session.commit() + if not result.rowcount: # type: ignore[attr-defined] + return None + return await self.get(operation_id) + + async def release_replacement_lease( + self, operation_id: int | None, *, owner: str + ) -> bool: + if operation_id is None: + return False + result = await self.session.execute( + update(RenameOperation) + .where( + col(RenameOperation.id) == operation_id, + col(RenameOperation.lease_owner) == owner, + ) + .values(lease_owner=None, lease_expires_at=None, updated_at=utc_now()) + .execution_options(synchronize_session="fetch") + ) + await self.session.commit() + return bool(result.rowcount) # type: ignore[attr-defined] + + async def mark_notified( + self, operation_id: int | None, when: datetime | None = None + ) -> bool: + """Set ``notified_at`` once; concurrent or repeated calls return False.""" + + if operation_id is None: + return False + notified_at = when or utc_now() + result = await self.session.execute( + update(RenameOperation) + .where( + col(RenameOperation.id) == operation_id, + col(RenameOperation.notified_at).is_(None), + ) + .values(notified_at=notified_at, updated_at=notified_at) + .execution_options(synchronize_session="fetch") + ) + await self.session.commit() + return bool(result.rowcount) # type: ignore[attr-defined] + + async def list_conflicts(self, limit: int = 100) -> list[RenameOperation]: + result = await self.session.execute( + select(RenameOperation) + .where(RenameOperation.state == "conflict") + .order_by(col(RenameOperation.updated_at).desc()) + .limit(limit) + ) + return list(result.scalars().all()) + + async def list_retryable( + self, now: datetime | None = None, limit: int = 100 + ) -> list[RenameOperation]: + ready_at = now or utc_now() + result = await self.session.execute( + select(RenameOperation) + .where( + RenameOperation.state == "retry", + or_( + col(RenameOperation.retry_at).is_(None), + col(RenameOperation.retry_at) <= ready_at, + ), + ) + .order_by(col(RenameOperation.retry_at).asc()) + .limit(limit) + ) + return list(result.scalars().all()) + + async def list_active_replacements(self, limit: int = 100) -> list[RenameOperation]: + result = await self.session.execute( + select(RenameOperation) + .where( + RenameOperation.kind == "replacement", + col(RenameOperation.state).in_( + ("planned", "old_staged", "new_promoted", "old_removed") + ), + ) + .order_by(col(RenameOperation.updated_at).asc()) + .limit(limit) + ) + return list(result.scalars().all()) + + async def delete(self, operation_id: int | None) -> bool: + row = await self.get(operation_id) + if row is None: + return False + await self.session.delete(row) + await self.session.commit() + return True + + async def prune_done(self, before: datetime) -> int: + result = await self.session.execute( + delete(RenameOperation) + .where( + col(RenameOperation.state) == "done", + col(RenameOperation.updated_at) < before, + ) + .execution_options(synchronize_session=False) + ) + await self.session.commit() + return int(result.rowcount or 0) # type: ignore[attr-defined] diff --git a/backend/src/module/downloader/__init__.py b/backend/src/module/downloader/__init__.py index 7d190d7c..abf2f5bb 100644 --- a/backend/src/module/downloader/__init__.py +++ b/backend/src/module/downloader/__init__.py @@ -3,6 +3,8 @@ from .base import ( CoreDownloaderClient, DownloaderCapabilities, DownloaderClient, + RenameOutcome, + RenameResult, ) from .download_client import DownloadClient, shutdown @@ -12,5 +14,7 @@ __all__ = [ "DownloadClient", "DownloaderCapabilities", "DownloaderClient", + "RenameOutcome", + "RenameResult", "shutdown", ] diff --git a/backend/src/module/downloader/base.py b/backend/src/module/downloader/base.py index 2ff76194..3256c025 100644 --- a/backend/src/module/downloader/base.py +++ b/backend/src/module/downloader/base.py @@ -20,6 +20,38 @@ class AddResult(Enum): FAILED = "failed" +class RenameOutcome(str, Enum): + """Downloader-independent outcome of a single file rename.""" + + RENAMED = "renamed" + ALREADY_APPLIED = "already_applied" + DESTINATION_EXISTS = "destination_exists" + RETRYABLE_FAILURE = "retryable_failure" + + +@dataclass(frozen=True) +class RenameResult: + """Structured rename result preserved across the downloader facade. + + ``bool(result)`` deliberately keeps the old success/failure behaviour while + callers migrate to inspecting :attr:`outcome`. It must never make a + collision or retryable failure truthy. + """ + + outcome: RenameOutcome + detail: str | None = None + + @property + def succeeded(self) -> bool: + return self.outcome in { + RenameOutcome.RENAMED, + RenameOutcome.ALREADY_APPLIED, + } + + def __bool__(self) -> bool: + return self.succeeded + + @dataclass(frozen=True) class DownloaderCapabilities: """What a concrete download client can do. @@ -83,6 +115,8 @@ class DownloaderClient(Protocol): async def torrents_info(self, status_filter, category, tag=None) -> list[dict]: ... + async def torrent_exists(self, torrent_hash: str) -> bool | None: ... + async def torrents_files(self, torrent_hash: str) -> list[dict]: ... async def torrents_delete(self, hash, delete_files: bool = True) -> bool: ... @@ -93,7 +127,7 @@ class DownloaderClient(Protocol): async def torrents_rename_file( self, torrent_hash, old_path, new_path, verify: bool = True - ) -> bool: ... + ) -> RenameResult: ... async def move_torrent(self, hashes, new_location) -> None: ... diff --git a/backend/src/module/downloader/client/aria2_downloader.py b/backend/src/module/downloader/client/aria2_downloader.py index 7d57d56d..c746298c 100644 --- a/backend/src/module/downloader/client/aria2_downloader.py +++ b/backend/src/module/downloader/client/aria2_downloader.py @@ -8,7 +8,13 @@ from typing import ClassVar import httpx from module.database import Database -from module.downloader.base import AddResult, DownloaderCapabilities +from module.database.aria2 import Aria2RenameIntent +from module.downloader.base import ( + AddResult, + DownloaderCapabilities, + RenameOutcome, + RenameResult, +) logger = logging.getLogger(__name__) @@ -220,6 +226,30 @@ class Aria2Downloader: os.makedirs(os.path.dirname(new_abs), exist_ok=True) shutil.move(old_abs, new_abs) + @staticmethod + def _rename_intent( + old_path: str, new_path: str, source_stat: os.stat_result + ) -> Aria2RenameIntent: + return Aria2RenameIntent( + old_path=old_path, + new_path=new_path, + st_dev=source_stat.st_dev, + st_ino=source_stat.st_ino, + st_size=source_stat.st_size, + st_mtime_ns=source_stat.st_mtime_ns, + ) + + @staticmethod + def _intent_matches_stat( + intent: Aria2RenameIntent, target_stat: os.stat_result + ) -> bool: + return ( + intent.st_dev == target_stat.st_dev + and intent.st_ino == target_stat.st_ino + and intent.st_size == target_stat.st_size + and intent.st_mtime_ns == target_stat.st_mtime_ns + ) + @staticmethod def _move_files(files: list[dict], old_dir: str, new_dir: str) -> None: old_dir = os.path.normpath(old_dir) @@ -430,7 +460,7 @@ class Aria2Downloader: stopped = await self._call("tellStopped", [0, 1000]) or [] except (Aria2RpcError, Aria2ConnectionError) as e: logger.error("Failed to query downloads: %s", e) - return [] + raise raw = [*active, *waiting, *stopped] followed_by: dict[str, str] = {} for d in raw: @@ -507,6 +537,20 @@ class Aria2Downloader: ) return result + async def torrent_exists(self, gid: str) -> bool | None: + """Return False only when aria2 explicitly confirms a gid is absent.""" + try: + await self._call("tellStatus", [gid, ["status"]]) + except Aria2RpcError as e: + if self._is_not_found_error(e): + return False + logger.warning("Cannot confirm whether aria2 gid %s exists: %s", gid, e) + return None + except Aria2ConnectionError as e: + logger.warning("Cannot confirm whether aria2 gid %s exists: %s", gid, e) + return None + return True + async def torrents_files(self, torrent_hash: str) -> list[dict]: try: files = await self._call("getFiles", [torrent_hash]) @@ -535,7 +579,7 @@ class Aria2Downloader: async def torrents_rename_file( self, torrent_hash, old_path, new_path, verify: bool = True - ) -> bool: + ) -> RenameResult: try: status = await self._call("tellStatus", [torrent_hash, ["dir"]]) except (Aria2RpcError, Aria2ConnectionError) as e: @@ -544,32 +588,118 @@ class Aria2Downloader: torrent_hash, e, ) - return False + return RenameResult(RenameOutcome.RETRYABLE_FAILURE, detail=str(e)) save_dir = (status or {}).get("dir", "") if not save_dir: - return False + return RenameResult( + RenameOutcome.RETRYABLE_FAILURE, + detail="aria2 returned no download directory", + ) old_abs = os.path.join(save_dir, old_path) new_abs = os.path.join(save_dir, new_path) + old_exists, new_exists = await asyncio.gather( + asyncio.to_thread(os.path.exists, old_abs), + asyncio.to_thread(os.path.exists, new_abs), + ) + async with Database() as db: + persisted_intent = await db.aria2.get_rename_intent(torrent_hash) + + if not old_exists: + matching_intent = ( + persisted_intent + if persisted_intent is not None + and persisted_intent.old_path == old_path + and persisted_intent.new_path == new_path + else None + ) + if not new_exists: + if matching_intent is not None: + async with Database() as db: + await db.aria2.clear_rename_intent( + torrent_hash, matching_intent + ) + return RenameResult( + RenameOutcome.RETRYABLE_FAILURE, + detail=f"rename source is missing: {old_abs}", + ) + if matching_intent is None: + return RenameResult( + RenameOutcome.DESTINATION_EXISTS, + detail=( + "destination exists without a matching durable rename " + f"intent: {new_abs}" + ), + ) + try: + target_stat = await asyncio.to_thread(os.stat, new_abs) + except OSError as e: + return RenameResult(RenameOutcome.RETRYABLE_FAILURE, detail=str(e)) + if not self._intent_matches_stat(matching_intent, target_stat): + async with Database() as db: + await db.aria2.clear_rename_intent(torrent_hash, matching_intent) + return RenameResult( + RenameOutcome.DESTINATION_EXISTS, + detail=f"destination does not match durable rename intent: {new_abs}", + ) + async with Database() as db: + finalized = await db.aria2.finalize_rename_intent( + torrent_hash, matching_intent + ) + if not finalized: + return RenameResult( + RenameOutcome.RETRYABLE_FAILURE, + detail="durable rename intent changed before recovery commit", + ) + return RenameResult(RenameOutcome.ALREADY_APPLIED) + + try: + source_stat = await asyncio.to_thread(os.stat, old_abs) + except OSError as e: + return RenameResult(RenameOutcome.RETRYABLE_FAILURE, detail=str(e)) + intent = self._rename_intent(old_path, new_path, source_stat) + async with Database() as db: + await db.aria2.set_rename_intent(torrent_hash, intent) try: await asyncio.to_thread(self._move_file, old_abs, new_abs) except FileExistsError: + async with Database() as db: + await db.aria2.clear_rename_intent(torrent_hash, intent) logger.warning( "Refusing to overwrite existing file %s, rename of %s skipped", new_abs, old_abs, ) - return False + return RenameResult( + RenameOutcome.DESTINATION_EXISTS, + detail=f"destination already exists: {new_abs}", + ) except OSError as e: + async with Database() as db: + await db.aria2.clear_rename_intent(torrent_hash, intent) logger.warning("Failed to rename file %s -> %s: %s", old_abs, new_abs, e) - return False - if verify: - exists = await asyncio.to_thread(os.path.exists, new_abs) - if not exists: - logger.debug("Rename reported success but %s is missing", new_abs) - return False + return RenameResult(RenameOutcome.RETRYABLE_FAILURE, detail=str(e)) + try: + target_stat = await asyncio.to_thread(os.stat, new_abs) + except OSError as e: + logger.debug("Rename reported success but %s cannot be verified", new_abs) + # Keep the intent: the move may have completed and a later retry can + # still reconcile it if the target becomes visible. + return RenameResult(RenameOutcome.RETRYABLE_FAILURE, detail=str(e)) + if not self._intent_matches_stat(intent, target_stat): + async with Database() as db: + await db.aria2.clear_rename_intent(torrent_hash, intent) + return RenameResult( + RenameOutcome.DESTINATION_EXISTS, + detail=f"renamed target identity changed unexpectedly: {new_abs}", + ) async with Database() as db: - await db.aria2.set_renamed_path(torrent_hash, old_path, new_path) - return True + finalized = await db.aria2.finalize_rename_intent(torrent_hash, intent) + if not finalized: + return RenameResult( + RenameOutcome.RETRYABLE_FAILURE, + detail="durable rename intent changed before sidecar commit", + ) + return RenameResult(RenameOutcome.RENAMED) # ------------------------------------------------------------------ # Management @@ -580,40 +710,98 @@ class Aria2Downloader: ok = True async with Database() as db: for gid in gids: - save_dir = None - try: - status = await self._call("tellStatus", [gid, ["dir"]]) - save_dir = (status or {}).get("dir") - except (Aria2RpcError, Aria2ConnectionError) as e: - logger.debug("Could not resolve dir for %s: %s", gid, e) - - if delete_files and save_dir: + if delete_files: try: + status = await self._call("tellStatus", [gid, ["dir"]]) + save_dir = (status or {}).get("dir") + if not save_dir: + raise Aria2ConnectionError( + f"aria2 returned no download directory for {gid}" + ) files = await self._call("getFiles", [gid]) - except (Aria2RpcError, Aria2ConnectionError): - files = [] + except Aria2RpcError as e: + if self._is_not_found_error(e): + # The task disappeared after a previous successful + # cleanup (or was removed externally). There is no + # downloader-owned file list left to resolve. + await db.aria2.delete(gid) + continue + logger.warning( + "Cannot safely delete current files for %s: %s", gid, e + ) + ok = False + continue + except Aria2ConnectionError as e: + # If the current paths cannot be enumerated safely, do + # not remove the task or its sidecar. A later retry can + # still reconcile the staged files. + logger.warning( + "Cannot safely delete current files for %s: %s", gid, e + ) + ok = False + continue + + renamed_paths = await db.aria2.get_renamed_paths(gid) + files_removed = True for f in files or []: - path = f.get("path") - if not path: + raw_path = f.get("path") + if not raw_path: + continue + relative_path = os.path.relpath(raw_path, save_dir) + current_relative_path = self._translate_renamed_path( + relative_path, renamed_paths + ) + current_path = os.path.normpath( + os.path.join(save_dir, current_relative_path) + ) + boundary = os.path.normpath(save_dir) + try: + within_boundary = ( + os.path.commonpath([boundary, current_path]) == boundary + ) + except ValueError: + within_boundary = False + if not within_boundary: + logger.warning( + "Refusing to delete aria2 path outside save dir: %s", + current_path, + ) + files_removed = False continue try: await asyncio.to_thread( - self._remove_file_and_empty_dirs, path, save_dir + self._remove_file_and_empty_dirs, + current_path, + save_dir, ) except OSError as e: - logger.warning("Failed to delete file %s: %s", path, e) + logger.warning( + "Failed to delete current file %s: %s", current_path, e + ) + files_removed = False + if not files_removed: + ok = False + continue + removed = False try: await self._call("forceRemove", [gid]) + removed = True except Aria2RpcError as e: - if not self._is_not_found_error(e): + if self._is_not_found_error(e): + removed = True + else: logger.error("Failed to remove %s: %s", gid, e) ok = False except Aria2ConnectionError as e: logger.error("Failed to remove %s: %s", gid, e) ok = False - await db.aria2.delete(gid) + # The local mapping is recovery state. Keep it whenever aria2 + # may still own the task; delete only after confirmed removal + # or an explicit not-found response. + if removed: + await db.aria2.delete(gid) return ok async def torrents_pause(self, hashes: str | list[str]) -> None: diff --git a/backend/src/module/downloader/client/mock_downloader.py b/backend/src/module/downloader/client/mock_downloader.py index 22002083..b9c8bde8 100644 --- a/backend/src/module/downloader/client/mock_downloader.py +++ b/backend/src/module/downloader/client/mock_downloader.py @@ -8,7 +8,12 @@ qBittorrent instance. All operations return success and log their actions. import logging from typing import Any, ClassVar -from module.downloader.base import AddResult, DownloaderCapabilities +from module.downloader.base import ( + AddResult, + DownloaderCapabilities, + RenameOutcome, + RenameResult, +) logger = logging.getLogger(__name__) @@ -92,6 +97,9 @@ class MockDownloader: logger.debug("get_torrents_by_tag(%s)", tag) return [t for t in self._torrents.values() if tag in t.get("tags", [])] + async def torrent_exists(self, torrent_hash: str) -> bool | None: + return torrent_hash in self._torrents + async def torrents_files(self, torrent_hash: str) -> list[dict]: """Return files for a torrent.""" logger.debug("torrents_files(%s)", torrent_hash) @@ -159,9 +167,9 @@ class MockDownloader: async def torrents_rename_file( self, torrent_hash: str, old_path: str, new_path: str, verify: bool = True - ) -> bool: + ) -> RenameResult: logger.info(f"rename: {old_path} -> {new_path}") - return True + return RenameResult(RenameOutcome.RENAMED) async def rss_add_feed(self, url: str, item_path: str): self._feeds[item_path] = {"url": url, "path": item_path} diff --git a/backend/src/module/downloader/client/qb_downloader.py b/backend/src/module/downloader/client/qb_downloader.py index b2379f23..f38a04eb 100644 --- a/backend/src/module/downloader/client/qb_downloader.py +++ b/backend/src/module/downloader/client/qb_downloader.py @@ -8,7 +8,12 @@ from typing import ClassVar import httpx from module.ab_decorator import qb_connect_failed_wait -from module.downloader.base import AddResult, DownloaderCapabilities +from module.downloader.base import ( + AddResult, + DownloaderCapabilities, + RenameOutcome, + RenameResult, +) logger = logging.getLogger(__name__) @@ -311,6 +316,28 @@ class QbDownloader: ] return torrents + async def torrent_exists(self, torrent_hash: str) -> bool | None: + """Query one hash without confusing a failed request with absence.""" + + try: + resp = await self._get("torrents/info", params={"hashes": torrent_hash}) + if resp.status_code >= 300: + logger.warning( + "Could not verify qBittorrent task %s: HTTP %s", + torrent_hash, + resp.status_code, + ) + return None + payload = resp.json() + except (ConnectionError, OSError, ValueError, httpx.RequestError) as e: + logger.warning("Could not verify qBittorrent task %s: %s", torrent_hash, e) + return None + return isinstance(payload, list) and any( + str(item.get("hash", "")).lower() == torrent_hash.lower() + for item in payload + if isinstance(item, dict) + ) + @qb_connect_failed_wait async def torrents_files(self, torrent_hash: str): resp = await self._get("torrents/files", params={"hash": torrent_hash}) @@ -555,7 +582,7 @@ class QbDownloader: async def torrents_rename_file( self, torrent_hash, old_path, new_path, verify: bool = True - ) -> bool: + ) -> RenameResult: try: resp = await self._post( "torrents/renameFile", @@ -563,13 +590,19 @@ class QbDownloader: ) if resp.status_code == 409: logger.debug("Conflict409Error: %s >> %s", old_path, new_path) - return False + return RenameResult( + RenameOutcome.DESTINATION_EXISTS, + detail="qBittorrent rejected rename with HTTP 409", + ) # qB 5.2 对成功的 renameFile 回 204（空响应体全局改为 204） if resp.status_code >= 300: - return False + return RenameResult( + RenameOutcome.RETRYABLE_FAILURE, + detail=f"qBittorrent rename returned HTTP {resp.status_code}", + ) if not verify: - return True + return RenameResult(RenameOutcome.RENAMED) # Verify the rename actually happened by checking file list # qBittorrent can return 200 but delay the actual rename (e.g., while seeding) @@ -579,7 +612,7 @@ class QbDownloader: await asyncio.sleep(delay) files = await self.torrents_files(torrent_hash) if any(f.get("name") == new_path for f in files): - return True + return RenameResult(RenameOutcome.RENAMED) # new_path 未出现（旧名仍在，或两个名字都不在列表里）一律不算 # 成功：盲目返回 True 会让 renamer 每周期重复改名 + 发通知 # (#754/#749)。返回 False 由调用方走 _PENDING_RENAME_COOLDOWN 退避。 @@ -590,10 +623,13 @@ class QbDownloader: new_path, old_path, ) - return False + return RenameResult( + RenameOutcome.RETRYABLE_FAILURE, + detail="qBittorrent did not expose the target path after rename", + ) except (httpx.ConnectError, httpx.RequestError, httpx.TimeoutException) as e: logger.warning(f"Failed to rename file {old_path}: {e}") - return False + return RenameResult(RenameOutcome.RETRYABLE_FAILURE, detail=str(e)) async def rss_add_feed(self, url, item_path): resp = await self._post( diff --git a/backend/src/module/downloader/download_client.py b/backend/src/module/downloader/download_client.py index 60c6fbdd..8652afaa 100644 --- a/backend/src/module/downloader/download_client.py +++ b/backend/src/module/downloader/download_client.py @@ -7,7 +7,7 @@ from module.conf import settings from module.models import Bangumi, Torrent from module.network import RequestContent -from .base import AddResult, DownloaderClient +from .base import AddResult, DownloaderClient, RenameOutcome, RenameResult from .path import gen_save_path logger = logging.getLogger(__name__) @@ -312,18 +312,51 @@ class DownloadClient: return [] return await self.client.torrents_files(torrent_hash=torrent_hash) + async def torrent_exists(self, torrent_hash: str) -> bool | None: + """Return task existence, or ``None`` when the backend cannot prove it. + + Destructive recovery paths must distinguish a confirmed absence from + a transient query failure; an empty bulk snapshot is not sufficient. + """ + + if not self._supports("can_query", "torrent_exists"): + return None + return await self.client.torrent_exists(torrent_hash) + async def rename_torrent_file( self, _hash, old_path, new_path, verify: bool = True - ) -> bool: + ) -> RenameResult: if not self._supports("can_rename", "rename_torrent_file"): - return False - result = await self.client.torrents_rename_file( + return RenameResult( + RenameOutcome.RETRYABLE_FAILURE, + detail=f"{type(self.client).__name__} does not support file rename", + ) + raw_result = await self.client.torrents_rename_file( torrent_hash=_hash, old_path=old_path, new_path=new_path, verify=verify ) - if result: + # Compatibility for test doubles and third-party clients that still + # implement the pre-3.3.4 boolean contract. Concrete built-in clients + # all return RenameResult, and the facade always exposes RenameResult. + if isinstance(raw_result, RenameResult): + result = raw_result + else: + result = RenameResult( + ( + RenameOutcome.RENAMED + if raw_result + else RenameOutcome.RETRYABLE_FAILURE + ), + detail=None if raw_result else "legacy downloader returned false", + ) + if result.succeeded: logger.info(f"{old_path} >> {new_path}") else: - logger.debug("Rename failed: %s >> %s", old_path, new_path) + logger.debug( + "Rename %s: %s >> %s", + result.outcome.value, + old_path, + new_path, + ) return result async def delete_torrent(self, hashes, delete_files: bool = True) -> bool: diff --git a/backend/src/module/manager/renamer.py b/backend/src/module/manager/renamer.py index 8506b0f7..87210607 100644 --- a/backend/src/module/manager/renamer.py +++ b/backend/src/module/manager/renamer.py @@ -1,8 +1,14 @@ import asyncio +import hashlib +import json import logging -import time +import uuid +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone from pathlib import PurePath +from sqlalchemy.exc import IntegrityError + from module.conf import settings from module.database import Database from module.database.bangumi import ( @@ -10,25 +16,49 @@ from module.database.bangumi import ( match_bangumi_in_list, normalize_save_path, ) -from module.downloader import DownloadClient +from module.downloader import DownloadClient, RenameOutcome, RenameResult from module.downloader.path import check_files, is_ep, path_to_bangumi -from module.models import EpisodeFile, Notification, SubtitleFile +from module.models import EpisodeFile, Notification, RenameOperation, SubtitleFile +from module.notification import RenameConflictEvent from module.parser import TitleParser +from .revision_policy import ( + RevisionIdentity, + is_strict_upgrade, + parse_revision_identity, + replacement_staged_path, +) + logger = logging.getLogger(__name__) -# Module-level cache to track pending renames that qBittorrent hasn't processed yet -# Key: (torrent_hash, old_path, new_path), Value: timestamp of last attempt -# This prevents spamming the same rename when qBittorrent returns 200 but doesn't actually rename -_pending_renames: dict[tuple[str, str, str], float] = {} _PENDING_RENAME_COOLDOWN = 300 # 5 minutes cooldown before retrying same rename -_CLEANUP_INTERVAL = 60 # Clean up pending cache at most once per minute -_last_cleanup_time: float = 0 # 处理完成标记：供外部脚本（filebot、hlink 等）过滤 AB 已重命名的任务 (#147)。 # 语义 = 顶层媒体文件全部就位；字幕在同一轮循环里紧随其后重命名，深层 # 嵌套文件（特典/花絮）设计上从不重命名——两者都不阻塞打标 _RENAMED_TAG = "ab:renamed" +_replacement_locks: dict[tuple[str, str, str], asyncio.Lock] = {} + + +@dataclass(frozen=True, slots=True) +class PreparedMediaRename: + episode: EpisodeFile + source_path: str + target_path: str + + +@dataclass(frozen=True, slots=True) +class MediaRenameReport: + result: RenameResult + prepared: PreparedMediaRename | None = None + notification: Notification | None = None + + +@dataclass(frozen=True, slots=True) +class RevisionOwner: + info: dict + files: list[dict] + identity: RevisionIdentity | None class Renamer: @@ -36,22 +66,7 @@ class Renamer: self.client = client self._parser = TitleParser() self._offset_cache: dict[str, tuple[int, int]] = {} - - @staticmethod - def _cleanup_pending_cache(): - """Clean up expired entries from pending renames cache (throttled).""" - global _last_cleanup_time - current_time = time.time() - if current_time - _last_cleanup_time < _CLEANUP_INTERVAL: - return - _last_cleanup_time = current_time - expired_keys = [ - k - for k, v in _pending_renames.items() - if current_time - v > _PENDING_RENAME_COOLDOWN * 2 - ] - for k in expired_keys: - _pending_renames.pop(k, None) + self.events: list[RenameConflictEvent] = [] @staticmethod def print_result(torrent_count, rename_count): @@ -178,61 +193,123 @@ class Renamer: existing_tags: str | None = None, **kwargs, ): + report = await self._rename_media_file( + torrent_name=torrent_name, + media_path=media_path, + bangumi_name=bangumi_name, + method=method, + season=season, + _hash=_hash, + episode_offset=episode_offset, + season_offset=season_offset, + episode_type=episode_type, + ) + if report.result.succeeded and method not in ("none", "normal"): + await self._mark_renamed(_hash, existing_tags) + return report.notification + + def _prepare_media_rename( + self, + *, + torrent_name: str, + media_path: str, + bangumi_name: str, + method: str, + season: int, + episode_offset: int = 0, + season_offset: int = 0, + episode_type: str = "episode", + ) -> PreparedMediaRename | None: ep = self._parser.torrent_parser( torrent_name=torrent_name, torrent_path=media_path, season=season, episode_type=episode_type, ) - if ep: - new_path = self.gen_path( + if ep is None: + return None + return PreparedMediaRename( + episode=ep, + source_path=media_path, + target_path=self.gen_path( ep, bangumi_name, method=method, episode_offset=episode_offset, season_offset=season_offset, - ) - if media_path == new_path: - # 已符合目标命名（如重启后再次扫描）：视为处理完成。 - # none/normal 是直通方法，没有"重命名完成"的语义，不打标 - if method not in ("none", "normal"): - await self._mark_renamed(_hash, existing_tags) - else: - # Check if this rename was recently attempted but didn't take effect - # (qBittorrent can return 200 but delay actual rename while seeding) - pending_key = (_hash, media_path, new_path) - last_attempt = _pending_renames.get(pending_key) - if ( - last_attempt - and (time.time() - last_attempt) < _PENDING_RENAME_COOLDOWN - ): - logger.debug("Skipping rename (pending cooldown): %s", media_path) - return None + ), + ) - if await self.client.rename_torrent_file( - _hash=_hash, old_path=media_path, new_path=new_path - ): - # Rename verified successful, remove from pending cache - _pending_renames.pop(pending_key, None) - await self._mark_renamed(_hash, existing_tags) - # Season comes from folder which already has offset applied - # Only apply episode offset - return Notification( - official_title=bangumi_name, - season=ep.season, - episode=self._adjust_episode(ep.episode, episode_offset), - ) - else: - # Rename API returned success but file wasn't actually renamed - # Add to pending cache to avoid spamming - _pending_renames[pending_key] = time.time() - # Periodic cleanup of expired entries (at most once per minute) - self._cleanup_pending_cache() - else: - logger.warning(f"{media_path} parse failed") + async def _execute_media_rename( + self, + *, + prepared: PreparedMediaRename, + bangumi_name: str, + _hash: str, + episode_offset: int, + ) -> MediaRenameReport: + if prepared.source_path == prepared.target_path: + return MediaRenameReport( + result=RenameResult(RenameOutcome.ALREADY_APPLIED), + prepared=prepared, + ) + result = await self.client.rename_torrent_file( + _hash=_hash, + old_path=prepared.source_path, + new_path=prepared.target_path, + ) + notification = None + if result.outcome is RenameOutcome.RENAMED: + notification = Notification( + official_title=bangumi_name, + season=prepared.episode.season, + episode=self._adjust_episode(prepared.episode.episode, episode_offset), + ) + return MediaRenameReport( + result=result, + prepared=prepared, + notification=notification, + ) + + async def _rename_media_file( + self, + *, + torrent_name: str, + media_path: str, + bangumi_name: str, + method: str, + season: int, + _hash: str, + episode_offset: int = 0, + season_offset: int = 0, + episode_type: str = "episode", + ) -> MediaRenameReport: + prepared = self._prepare_media_rename( + torrent_name=torrent_name, + media_path=media_path, + bangumi_name=bangumi_name, + method=method, + season=season, + episode_offset=episode_offset, + season_offset=season_offset, + episode_type=episode_type, + ) + if prepared is None: + logger.warning("%s parse failed", media_path) if settings.bangumi_manage.remove_bad_torrent: await self.client.delete_torrent(hashes=_hash) - return None + return MediaRenameReport( + result=RenameResult( + RenameOutcome.RETRYABLE_FAILURE, + detail="media path could not be parsed", + ) + ) + return await self._execute_media_rename( + prepared=prepared, + bangumi_name=bangumi_name, + _hash=_hash, + episode_offset=episode_offset, + ) @staticmethod def _gen_movie_extra_path(new_path: str, media_path: str) -> str: @@ -288,6 +365,8 @@ class Renamer: episode_type: str = "episode", file_sizes: dict[str, int] | None = None, existing_tags: str | None = None, + mark_complete: bool = True, + torrent_info: dict | None = None, **kwargs, ): # 多文件电影种子（正片 + 特典/花絮）：所有文件会解析出同一标题， @@ -324,23 +403,43 @@ class Renamer: # new_path == media_path 说明是 none 等直通方法，不做区分 new_path = self._gen_movie_extra_path(new_path, media_path) if media_path != new_path: - renamed = await self.client.rename_torrent_file( - _hash=_hash, old_path=media_path, new_path=new_path + prepared = PreparedMediaRename( + episode=ep, + source_path=media_path, + target_path=new_path, ) - if not renamed: + if torrent_info is not None: + identity = parse_revision_identity( + torrent_info.get("name", ""), + bangumi_id=self._parse_bangumi_id_from_tags( + torrent_info.get("tags") + ), + default_season=season, + episode_offset=episode_offset, + ) + report = await self._run_ordinary_rename( + info=torrent_info, + prepared=prepared, + identity=identity, + bangumi_name=bangumi_name, + episode_offset=episode_offset, + ) + result = report.result + else: + result = await self.client.rename_torrent_file( + _hash=_hash, + old_path=media_path, + new_path=new_path, + ) + if not result.succeeded: all_renamed = False logger.warning(f"{media_path} rename failed") - # Delete bad torrent. - if settings.bangumi_manage.remove_bad_torrent: - await self.client.delete_torrent(_hash) - break else: # 解析失败的媒体文件不会被重命名——不能算处理完成 all_renamed = False - # 全部媒体文件就位（含本轮无需改名的情况）才算处理完成； - # none/normal 直通方法没有"重命名完成"的语义，不打标 - if all_renamed and method not in ("none", "normal"): + if all_renamed and mark_complete and method not in ("none", "normal"): await self._mark_renamed(_hash, existing_tags) + return all_renamed async def rename_subtitles( self, @@ -400,6 +499,1028 @@ class Renamer: pass return None + @staticmethod + def _has_tag(tags: str | None, expected: str) -> bool: + return expected in (tag.strip() for tag in (tags or "").split(",")) + + @staticmethod + def _retry_at() -> datetime: + return datetime.now(timezone.utc) + timedelta(seconds=_PENDING_RENAME_COOLDOWN) + + @staticmethod + def _retry_is_due(value: datetime | None) -> bool: + if value is None: + return True + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value <= datetime.now(timezone.utc) + + def _downloader_type(self) -> str: + configured = settings.downloader.type + downloader_type = ( + configured + if isinstance(configured, str) + else type(self.client.client).__name__.lower() + ) + host = settings.downloader.host + if not isinstance(host, str) or not host: + return downloader_type + instance_hash = hashlib.sha256(host.strip().lower().encode()).hexdigest()[:12] + return f"{downloader_type}:{instance_hash}" + + async def _find_revision_owners( + self, + *, + incoming: dict, + target_path: str, + all_infos: list[dict], + episode_offset: int, + ) -> tuple[RevisionIdentity | None, list[RevisionOwner]]: + """Find downloader tasks that already own an incoming canonical path.""" + + incoming_id = self._parse_bangumi_id_from_tags(incoming.get("tags")) + _, incoming_season = path_to_bangumi( + incoming.get("save_path", ""), incoming.get("name", "") + ) + incoming_identity = parse_revision_identity( + incoming.get("name", ""), + bangumi_id=incoming_id, + default_season=incoming_season, + episode_offset=episode_offset, + ) + if incoming_id is None: + return incoming_identity, [] + + save_path = normalize_save_path(incoming.get("save_path", "")) + candidates = [ + info + for info in all_infos + if info.get("hash") != incoming.get("hash") + and normalize_save_path(info.get("save_path", "")) == save_path + and self._parse_bangumi_id_from_tags(info.get("tags")) == incoming_id + ] + if not candidates: + return incoming_identity, [] + + candidate_files = await asyncio.gather( + *[self.client.get_torrent_files(info["hash"]) for info in candidates] + ) + owners: list[RevisionOwner] = [] + normalized_target = target_path.replace("\\", "/") + for info, files in zip(candidates, candidate_files): + # Count every downloader task that references the canonical path + # before applying the destructive single-file guard. A collection + # owner must make replacement ineligible, not disappear from the + # owner count. + if not any( + str(item.get("name", "")).replace("\\", "/") == normalized_target + for item in files + ): + continue + _, old_season = path_to_bangumi( + info.get("save_path", ""), info.get("name", "") + ) + owners.append( + RevisionOwner( + info=info, + files=files, + identity=parse_revision_identity( + info.get("name", ""), + bangumi_id=incoming_id, + default_season=old_season, + episode_offset=episode_offset, + ), + ) + ) + return incoming_identity, owners + + def _build_operation( + self, + *, + info: dict, + prepared: PreparedMediaRename, + identity: RevisionIdentity | None, + kind: str, + state: str, + owner: RevisionOwner | None = None, + reason: str | None = None, + ) -> RenameOperation: + owner_identity = owner.identity if owner else None + metadata = { + "new_torrent_name": info.get("name", ""), + "old_torrent_name": owner.info.get("name", "") if owner else None, + } + return RenameOperation( + downloader_type=self._downloader_type(), + kind=kind, + state=state, + new_task_id=info["hash"], + old_task_id=owner.info["hash"] if owner else None, + save_path=normalize_save_path(info.get("save_path", "")), + source_path=prepared.source_path, + target_path=prepared.target_path, + staged_path=( + replacement_staged_path( + prepared.target_path, + old_task_id=owner.info["hash"], + old_revision=owner_identity.revision, + ) + if owner and owner_identity + else None + ), + bangumi_id=identity.bangumi_id if identity else None, + media_type=identity.media_type.value if identity else None, + season=identity.season if identity else None, + episode=float(identity.episode) if identity else None, + group_name=identity.group if identity else None, + resolution=identity.resolution if identity else None, + old_revision=owner_identity.revision if owner_identity else None, + new_revision=identity.revision if identity else None, + revision_metadata=json.dumps(metadata, ensure_ascii=False), + last_error=reason, + ) + + async def _emit_conflict_once( + self, + operation: RenameOperation, + *, + torrent_name: str, + reason: str, + ) -> None: + async with Database() as db: + claimed = await db.rename_operation.mark_notified(operation.id) + if claimed: + self.events.append( + RenameConflictEvent( + task_id=operation.new_task_id, + torrent_name=torrent_name, + target_path=operation.target_path, + reason=reason, + ) + ) + + async def _persist_conflict( + self, + *, + info: dict, + prepared: PreparedMediaRename, + identity: RevisionIdentity | None, + reason: str, + owner: RevisionOwner | None = None, + ) -> RenameOperation | None: + operation = self._build_operation( + info=info, + prepared=prepared, + identity=identity, + kind="conflict", + state="conflict", + owner=owner, + reason=reason, + ) + async with Database() as db: + active = await db.rename_operation.get_by_target( + downloader_type=operation.downloader_type, + save_path=operation.save_path, + target_path=operation.target_path, + ) + if active is not None and active.new_task_id != operation.new_task_id: + logger.warning( + "Rename target already reserved by task %s: %s", + active.new_task_id[:8], + operation.target_path, + ) + return active + try: + row, _ = await db.rename_operation.upsert_conflict(operation) + except IntegrityError: + row = await db.rename_operation.get_by_target( + downloader_type=operation.downloader_type, + save_path=operation.save_path, + target_path=operation.target_path, + ) + if row is not None and row.new_task_id == info["hash"]: + await self._emit_conflict_once( + row, torrent_name=info.get("name", ""), reason=reason + ) + return row + + async def _set_operation_state( + self, + operation: RenameOperation, + state: str, + *, + retry: bool = False, + error: str | None = None, + ) -> RenameOperation: + async with Database() as db: + if operation.kind == "replacement" and operation.lease_owner: + updated = await db.rename_operation.set_state_claimed( + operation.id, + owner=operation.lease_owner, + state=state, # type: ignore[arg-type] + retry_at=self._retry_at() if retry else None, + last_error=error, + ) + if updated is None: + raise RuntimeError("replacement lease was lost before state commit") + else: + updated = await db.rename_operation.set_state( + operation.id, + state, # type: ignore[arg-type] + retry_at=self._retry_at() if retry else None, + last_error=error, + ) + return updated or operation + + async def _replacement_conflict( + self, + operation: RenameOperation, + *, + info: dict, + reason: str, + ) -> None: + operation = await self._set_operation_state(operation, "conflict", error=reason) + await self._emit_conflict_once( + operation, + torrent_name=info.get("name", ""), + reason=reason, + ) + + async def _advance_replacement( + self, + *, + operation: RenameOperation, + info: dict, + prepared: PreparedMediaRename, + all_infos: list[dict], + bangumi_name: str, + episode_offset: int, + existing_tags: str | None, + ) -> Notification | None: + """Advance a staged V1 -> V2 replacement saga as far as possible.""" + + key = ( + operation.downloader_type, + operation.save_path, + operation.target_path, + ) + lock = _replacement_locks.setdefault(key, asyncio.Lock()) + async with lock: + async with Database() as db: + current = await db.rename_operation.get(operation.id) + if current is None: + return None + operation = current + if not self._retry_is_due(operation.retry_at): + return None + + info_by_hash = {item["hash"]: item for item in all_infos} + staged_path = operation.staged_path + if not staged_path or not operation.old_task_id: + await self._replacement_conflict( + operation, + info=info, + reason="replacement operation is missing old owner metadata", + ) + return None + old_task_id = operation.old_task_id + + # A bounded loop lets the normal successful path finish in one tick, + # while every external action is verified and persisted separately. + for _ in range(6): + state = operation.state + if state in { + "planned", + "old_staged", + "new_promoted", + "old_removed", + }: + async with Database() as db: + claimed = await db.rename_operation.claim_replacement_lease( + operation.id, + owner=uuid.uuid4().hex, + ) + if claimed is None: + return None + operation = claimed + state = operation.state + if state == "conflict": + await self._emit_conflict_once( + operation, + torrent_name=info.get("name", ""), + reason=operation.last_error or "rename conflict", + ) + return None + + if state == "planned": + old_info = info_by_hash.get(old_task_id) + if old_info is None: + await self._replacement_conflict( + operation, + info=info, + reason="old revision task disappeared before staging", + ) + return None + old_files = await self.client.get_torrent_files(old_task_id) + old_names = { + str(item.get("name", "")).replace("\\", "/") + for item in old_files + } + if staged_path in old_names: + operation = await self._set_operation_state( + operation, "old_staged" + ) + continue + if operation.target_path not in old_names: + await self._replacement_conflict( + operation, + info=info, + reason="old revision no longer owns the canonical path", + ) + return None + result = await self.client.rename_torrent_file( + _hash=old_task_id, + old_path=operation.target_path, + new_path=staged_path, + ) + if result.succeeded: + operation = await self._set_operation_state( + operation, "old_staged" + ) + continue + if result.outcome is RenameOutcome.DESTINATION_EXISTS: + await self._replacement_conflict( + operation, + info=info, + reason="temporary staging path already exists", + ) + return None + operation = await self._set_operation_state( + operation, + "planned", + retry=True, + error=result.detail or "failed to stage old revision", + ) + return None + + if state == "old_staged": + new_files = await self.client.get_torrent_files( + operation.new_task_id + ) + new_names = { + str(item.get("name", "")).replace("\\", "/") + for item in new_files + } + if operation.target_path in new_names: + operation = await self._set_operation_state( + operation, "new_promoted" + ) + continue + if operation.source_path not in new_names: + rollback = await self.client.rename_torrent_file( + _hash=old_task_id, + old_path=staged_path, + new_path=operation.target_path, + ) + if rollback.succeeded: + await self._replacement_conflict( + operation, + info=info, + reason=( + "new revision source disappeared after " + "staging; V1 was restored" + ), + ) + else: + await self._set_operation_state( + operation, + "old_staged", + retry=True, + error=( + "new revision source disappeared and V1 " + "rollback needs retry: " + f"{rollback.detail or rollback.outcome.value}" + ), + ) + return None + result = await self.client.rename_torrent_file( + _hash=operation.new_task_id, + old_path=operation.source_path, + new_path=operation.target_path, + ) + if result.succeeded: + operation = await self._set_operation_state( + operation, "new_promoted" + ) + continue + + # Reconcile once more before rollback: a transport failure + # may happen after the downloader applied the promotion. + new_files = await self.client.get_torrent_files( + operation.new_task_id + ) + if any( + str(item.get("name", "")).replace("\\", "/") + == operation.target_path + for item in new_files + ): + operation = await self._set_operation_state( + operation, "new_promoted" + ) + continue + + rollback = await self.client.rename_torrent_file( + _hash=old_task_id, + old_path=staged_path, + new_path=operation.target_path, + ) + if rollback.succeeded: + if result.outcome is RenameOutcome.DESTINATION_EXISTS: + await self._replacement_conflict( + operation, + info=info, + reason=( + "V2 promotion target is owned by an unknown " + "file; V1 was restored" + ), + ) + return None + operation = await self._set_operation_state( + operation, + "planned", + retry=True, + error=result.detail + or "promotion failed and V1 was restored", + ) + return None + if rollback.outcome is RenameOutcome.DESTINATION_EXISTS: + await self._replacement_conflict( + operation, + info=info, + reason=( + "V2 promotion and V1 rollback both found an " + "occupied canonical target" + ), + ) + return None + operation = await self._set_operation_state( + operation, + "old_staged", + retry=True, + error=( + "promotion failed and rollback needs retry: " + f"{rollback.detail or rollback.outcome.value}" + ), + ) + return None + + if state == "new_promoted": + old_exists = await self.client.torrent_exists(old_task_id) + if old_exists is None: + operation = await self._set_operation_state( + operation, + "new_promoted", + retry=True, + error=( + "new revision is live; old task existence could " + "not be verified" + ), + ) + return None + if old_exists: + removed = await self.client.delete_torrent( + old_task_id, delete_files=True + ) + if not removed: + operation = await self._set_operation_state( + operation, + "new_promoted", + retry=True, + error="new revision is live; old task cleanup failed", + ) + return None + old_exists = await self.client.torrent_exists(old_task_id) + if old_exists is not False: + operation = await self._set_operation_state( + operation, + "new_promoted", + retry=True, + error=( + "old task deletion was accepted but is not " + "yet observable" + ), + ) + return None + operation = await self._set_operation_state( + operation, "old_removed" + ) + continue + + if state == "old_removed": + operation = await self._set_operation_state(operation, "done") + continue + + if state == "done": + await self._mark_renamed(info["hash"], existing_tags) + async with Database() as db: + notify = await db.rename_operation.mark_notified(operation.id) + if not notify: + return None + return Notification( + official_title=bangumi_name, + season=prepared.episode.season, + episode=self._adjust_episode( + prepared.episode.episode, episode_offset + ), + ) + + # `retry` is only used by ordinary renames. A replacement row + # reaching it is malformed and must stop rather than guessing. + await self._replacement_conflict( + operation, + info=info, + reason=f"unexpected replacement state: {state}", + ) + return None + return None + + async def _start_replacement( + self, + *, + info: dict, + prepared: PreparedMediaRename, + identity: RevisionIdentity, + owner: RevisionOwner, + all_infos: list[dict], + bangumi_name: str, + episode_offset: int, + existing_tags: str | None, + ) -> Notification | None: + operation = self._build_operation( + info=info, + prepared=prepared, + identity=identity, + kind="replacement", + state="planned", + owner=owner, + ) + async with Database() as db: + try: + row, _ = await db.rename_operation.get_or_create(operation) + except IntegrityError: + row = await db.rename_operation.get_by_target( + downloader_type=operation.downloader_type, + save_path=operation.save_path, + target_path=operation.target_path, + ) + if row is None or row.new_task_id != info["hash"]: + await self._persist_conflict( + info=info, + prepared=prepared, + identity=identity, + owner=owner, + reason="canonical target is reserved by another rename operation", + ) + return None + return await self._advance_replacement( + operation=row, + info=info, + prepared=prepared, + all_infos=all_infos, + bangumi_name=bangumi_name, + episode_offset=episode_offset, + existing_tags=existing_tags, + ) + + async def _recover_missing_replacement( + self, operation: RenameOperation, all_infos: list[dict] + ) -> None: + """Recover an active saga whose incoming task left the normal snapshot.""" + + if any(info.get("hash") == operation.new_task_id for info in all_infos): + return + async with Database() as db: + claimed = await db.rename_operation.claim_replacement_lease( + operation.id, + owner=uuid.uuid4().hex, + ) + if claimed is None: + return + operation = claimed + try: + metadata = json.loads(operation.revision_metadata or "{}") + except json.JSONDecodeError: + metadata = {} + info = { + "hash": operation.new_task_id, + "name": metadata.get("new_torrent_name") or operation.new_task_id, + } + if operation.state == "old_staged" and operation.old_task_id: + rollback = await self.client.rename_torrent_file( + _hash=operation.old_task_id, + old_path=operation.staged_path or "", + new_path=operation.target_path, + ) + if rollback.succeeded: + await self._replacement_conflict( + operation, + info=info, + reason="incoming revision task disappeared; V1 was restored", + ) + return + if rollback.outcome is RenameOutcome.DESTINATION_EXISTS: + await self._replacement_conflict( + operation, + info=info, + reason=( + "incoming revision task disappeared while the canonical " + "path remained occupied" + ), + ) + return + await self._set_operation_state( + operation, + "old_staged", + retry=True, + error=( + "incoming revision task disappeared and V1 rollback needs " + f"retry: {rollback.detail or rollback.outcome.value}" + ), + ) + return + if operation.state == "old_removed": + await self._set_operation_state(operation, "done") + return + await self._replacement_conflict( + operation, + info=info, + reason=( + "incoming revision task disappeared during replacement; " + "automatic deletion is stopped" + ), + ) + + async def _run_ordinary_rename( + self, + *, + info: dict, + prepared: PreparedMediaRename, + identity: RevisionIdentity | None, + bangumi_name: str, + episode_offset: int, + ) -> MediaRenameReport: + """Claim and execute one non-replacement rename exactly once at a time.""" + + template = self._build_operation( + info=info, + prepared=prepared, + identity=identity, + kind="conflict", + state="retry", + ) + key = (template.downloader_type, template.save_path, template.target_path) + lock = _replacement_locks.setdefault(key, asyncio.Lock()) + async with lock: + async with Database() as db: + active = await db.rename_operation.get_by_target( + downloader_type=template.downloader_type, + save_path=template.save_path, + target_path=template.target_path, + ) + if active is not None and active.new_task_id != info["hash"]: + return MediaRenameReport( + result=RenameResult( + RenameOutcome.DESTINATION_EXISTS, + detail="canonical target is reserved by another operation", + ), + prepared=prepared, + ) + if active is None: + try: + active, _ = await db.rename_operation.get_or_create(template) + except IntegrityError: + active = await db.rename_operation.get_by_target( + downloader_type=template.downloader_type, + save_path=template.save_path, + target_path=template.target_path, + ) + if active is None: + return MediaRenameReport( + result=RenameResult( + RenameOutcome.RETRYABLE_FAILURE, + detail="could not reserve rename operation", + ), + prepared=prepared, + ) + if active.state == "done": + return MediaRenameReport( + result=RenameResult(RenameOutcome.ALREADY_APPLIED), + prepared=prepared, + ) + if active.state == "conflict": + conflict = active + claimed = None + else: + conflict = None + if active.state == "running": + recovered = await db.rename_operation.recover_stale_running( + active.id, + before=datetime.now(timezone.utc) + - timedelta(seconds=_PENDING_RENAME_COOLDOWN), + ) + if not recovered: + return MediaRenameReport( + result=RenameResult( + RenameOutcome.RETRYABLE_FAILURE, + detail="rename operation is already running", + ), + prepared=prepared, + ) + active = await db.rename_operation.get(active.id) or active + if active.state == "retry" and not self._retry_is_due( + active.retry_at + ): + return MediaRenameReport( + result=RenameResult( + RenameOutcome.RETRYABLE_FAILURE, + detail=active.last_error or "rename retry cooldown", + ), + prepared=prepared, + ) + claimed = await db.rename_operation.claim( + active.id, + from_states=("retry",), + to_state="running", + ) + + if conflict is not None: + await self._emit_conflict_once( + conflict, + torrent_name=info.get("name", ""), + reason=conflict.last_error or "target already exists", + ) + return MediaRenameReport( + result=RenameResult( + RenameOutcome.DESTINATION_EXISTS, + detail=conflict.last_error, + ), + prepared=prepared, + ) + if claimed is None: + return MediaRenameReport( + result=RenameResult( + RenameOutcome.RETRYABLE_FAILURE, + detail="rename operation was claimed by another worker", + ), + prepared=prepared, + ) + + # The downloader may have applied the previous rename just before + # the process crashed, leaving the DB row in ``running``. Its own + # file list is authoritative proof of ownership; reconcile that + # state before sending the external mutation again (qB commonly + # answers the replay with 409). + current_files = await self.client.get_torrent_files(info["hash"]) + current_names = { + str(item.get("name", "")).replace("\\", "/") for item in current_files + } + if ( + prepared.target_path in current_names + and prepared.source_path not in current_names + ): + await self._set_operation_state(claimed, "done") + return MediaRenameReport( + result=RenameResult(RenameOutcome.ALREADY_APPLIED), + prepared=prepared, + ) + + report = await self._execute_media_rename( + prepared=prepared, + bangumi_name=bangumi_name, + _hash=info["hash"], + episode_offset=episode_offset, + ) + if report.result.succeeded: + await self._set_operation_state(claimed, "done") + return report + if report.result.outcome is RenameOutcome.DESTINATION_EXISTS: + conflict = await self._set_operation_state( + claimed, + "conflict", + error=report.result.detail or "target path already exists", + ) + await self._emit_conflict_once( + conflict, + torrent_name=info.get("name", ""), + reason=conflict.last_error or "target path already exists", + ) + return report + await self._set_operation_state( + claimed, + "retry", + retry=True, + error=report.result.detail or "rename failed verification", + ) + return report + + async def _process_single_torrent( + self, + *, + info: dict, + files: list[dict], + media_path: str, + all_infos: list[dict], + bangumi_name: str, + season: int, + method: str, + episode_offset: int, + season_offset: int, + episode_type: str, + ) -> MediaRenameReport: + prepared = self._prepare_media_rename( + torrent_name=info["name"], + media_path=media_path, + bangumi_name=bangumi_name, + method=method, + season=season, + episode_offset=episode_offset, + season_offset=season_offset, + episode_type=episode_type, + ) + if prepared is None: + logger.warning("%s parse failed", media_path) + if settings.bangumi_manage.remove_bad_torrent: + await self.client.delete_torrent(hashes=info["hash"]) + return MediaRenameReport( + result=RenameResult( + RenameOutcome.RETRYABLE_FAILURE, + detail="media path could not be parsed", + ) + ) + + incoming_id = self._parse_bangumi_id_from_tags(info.get("tags")) + identity = parse_revision_identity( + info.get("name", ""), + bangumi_id=incoming_id, + default_season=season, + episode_offset=episode_offset, + ) + save_path = normalize_save_path(info.get("save_path", "")) + async with Database() as db: + active = await db.rename_operation.get_by_target( + downloader_type=self._downloader_type(), + save_path=save_path, + target_path=prepared.target_path, + ) + + if active is not None: + if active.new_task_id != info["hash"]: + return MediaRenameReport( + result=RenameResult( + RenameOutcome.DESTINATION_EXISTS, + detail="canonical target is reserved by another operation", + ), + prepared=prepared, + ) + if active.state == "conflict": + await self._emit_conflict_once( + active, + torrent_name=info.get("name", ""), + reason=active.last_error or "target already exists", + ) + return MediaRenameReport( + result=RenameResult( + RenameOutcome.DESTINATION_EXISTS, + detail=active.last_error, + ), + prepared=prepared, + ) + if active.kind == "replacement" and active.state != "done": + notification = await self._advance_replacement( + operation=active, + info=info, + prepared=prepared, + all_infos=all_infos, + bangumi_name=bangumi_name, + episode_offset=episode_offset, + existing_tags=info.get("tags"), + ) + async with Database() as db: + refreshed = await db.rename_operation.get(active.id) + finished = refreshed is not None and refreshed.state == "done" + return MediaRenameReport( + result=RenameResult( + ( + RenameOutcome.RENAMED + if finished + else RenameOutcome.RETRYABLE_FAILURE + ), + detail=( + None + if finished + else (refreshed.last_error if refreshed else None) + ), + ), + prepared=prepared, + notification=notification, + ) + if active.state == "retry" and not self._retry_is_due(active.retry_at): + return MediaRenameReport( + result=RenameResult( + RenameOutcome.RETRYABLE_FAILURE, + detail=active.last_error or "rename retry cooldown", + ), + prepared=prepared, + ) + if active.state == "done": + return MediaRenameReport( + result=RenameResult(RenameOutcome.ALREADY_APPLIED), + prepared=prepared, + ) + + incoming_identity, owners = await self._find_revision_owners( + incoming=info, + target_path=prepared.target_path, + all_infos=all_infos, + episode_offset=episode_offset, + ) + if owners: + owner = owners[0] if len(owners) == 1 else None + reason = "canonical path has more than one downloader owner" + can_replace = bool( + owner is not None + and len(files) == 1 + and len(owner.files) == 1 + and incoming_identity is not None + and owner.identity is not None + and is_strict_upgrade(owner.identity, incoming_identity) + ) + if ( + settings.bangumi_manage.revision_conflict_policy == "replace" + and can_replace + ): + assert owner is not None + assert incoming_identity is not None + notification = await self._start_replacement( + info=info, + prepared=prepared, + identity=incoming_identity, + owner=owner, + all_infos=all_infos, + bangumi_name=bangumi_name, + episode_offset=episode_offset, + existing_tags=info.get("tags"), + ) + async with Database() as db: + replacement = await db.rename_operation.get_by_target( + downloader_type=self._downloader_type(), + save_path=save_path, + target_path=prepared.target_path, + ) + finished = replacement is not None and replacement.state == "done" + return MediaRenameReport( + result=RenameResult( + ( + RenameOutcome.RENAMED + if finished + else RenameOutcome.RETRYABLE_FAILURE + ), + detail=(replacement.last_error if replacement else None), + ), + prepared=prepared, + notification=notification, + ) + + if len(owners) == 1: + assert owner is not None + if len(files) != 1 or len(owner.files) != 1: + reason = "automatic replacement requires two single-file torrents" + elif incoming_identity is None or owner.identity is None: + reason = "revision identity is incomplete" + elif not is_strict_upgrade(owner.identity, incoming_identity): + reason = "existing and incoming releases are not a strict revision upgrade" + else: + reason = "revision conflict policy is hold" + await self._persist_conflict( + info=info, + prepared=prepared, + identity=incoming_identity, + owner=owner, + reason=reason, + ) + return MediaRenameReport( + result=RenameResult(RenameOutcome.DESTINATION_EXISTS, detail=reason), + prepared=prepared, + ) + + return await self._run_ordinary_rename( + info=info, + prepared=prepared, + identity=identity, + bangumi_name=bangumi_name, + episode_offset=episode_offset, + ) + @staticmethod def _normalize_path(path: str) -> str: """Normalize path by removing trailing slashes and standardizing separators.""" @@ -611,16 +1732,54 @@ class Renamer: return 0, 0 async def rename(self) -> list[Notification]: - # Get torrent info logger.debug("Start rename process.") rename_method = settings.bangumi_manage.rename_method - torrents_info = await self.client.get_torrent_info() + pending_infos = await self.client.get_torrent_info() + # Owner counting and Saga recovery must see tasks outside the normal + # Bangumi/completed filter (collections, paused tasks, changed category). + all_infos = await self.client.get_torrent_info( + category=None, status_filter=None + ) + info_by_hash = {info["hash"]: info for info in all_infos} + for info in pending_infos: + info_by_hash.setdefault(info["hash"], info) + all_infos = list(info_by_hash.values()) + async with Database() as db: + active_replacements = await db.rename_operation.list_active_replacements() + active_replacement_ids = { + operation.new_task_id for operation in active_replacements + } + for operation in active_replacements: + if operation.new_task_id not in info_by_hash: + incoming_exists = await self.client.torrent_exists( + operation.new_task_id + ) + if incoming_exists is None or incoming_exists: + # A failed/incomplete bulk snapshot must never be treated + # as proof that the incoming task disappeared. + continue + await self._recover_missing_replacement(operation, all_infos) + continue + if not any( + info.get("hash") == operation.new_task_id for info in pending_infos + ): + pending_infos.append(info_by_hash[operation.new_task_id]) + # `ab:renamed` is a real terminal marker. Filter it before file and + # offset queries; a later V2 still sees it lazily as a possible owner. + torrents_info = [ + info + for info in pending_infos + if info.get("hash") in active_replacement_ids + or not self._has_tag(info.get("tags"), _RENAMED_TAG) + ] renamed_info: list[Notification] = [] - # Fetch all torrent files concurrently + if not torrents_info: + logger.debug("Rename process finished: no pending torrents") + return renamed_info + all_files = await asyncio.gather( *[self.client.get_torrent_files(info["hash"]) for info in torrents_info] ) - # Batch lookup all offsets in a single database session offset_map = await self._batch_lookup_offsets(torrents_info) for info, files in zip(torrents_info, all_files): torrent_hash = info["hash"] @@ -637,7 +1796,6 @@ class Renamer: continue media_list, subtitle_list = check_files(files) bangumi_name, season = path_to_bangumi(save_path, torrent_name) - # Use pre-fetched offsets episode_offset, season_offset, episode_type = offset_map[torrent_hash] kwargs = { "torrent_name": torrent_name, @@ -648,29 +1806,51 @@ class Renamer: "episode_offset": episode_offset, "season_offset": season_offset, "episode_type": episode_type, - # 处理完成后打 ab:renamed 标签用；已带标签则跳过 API 调用 "existing_tags": info.get("tags"), } - # Rename single media file if len(media_list) == 1: - notify_info = await self.rename_file(media_path=media_list[0], **kwargs) - if notify_info: - renamed_info.append(notify_info) - # Rename subtitle file - if len(subtitle_list) > 0: - await self.rename_subtitles(subtitle_list=subtitle_list, **kwargs) - # Rename collection + report = await self._process_single_torrent( + info=info, + files=files, + media_path=media_list[0], + all_infos=all_infos, + bangumi_name=bangumi_name, + season=season, + method=rename_method, + episode_offset=episode_offset, + season_offset=season_offset, + episode_type=episode_type, + ) + if report.notification: + renamed_info.append(report.notification) + if report.result.succeeded: + if subtitle_list: + await self.rename_subtitles( + subtitle_list=subtitle_list, **kwargs + ) + if rename_method not in ("none", "normal"): + await self._mark_renamed(torrent_hash, info.get("tags")) elif len(media_list) > 1: logger.info("Start rename collection") - # 传入各文件体积，供多文件电影种子选出正片主文件 file_sizes = {f["name"]: f.get("size") or 0 for f in files} - await self.rename_collection( - media_list=media_list, file_sizes=file_sizes, **kwargs + collection_complete = await self.rename_collection( + media_list=media_list, + file_sizes=file_sizes, + mark_complete=False, + torrent_info=info, + **kwargs, ) - if len(subtitle_list) > 0: + if collection_complete and subtitle_list: await self.rename_subtitles(subtitle_list=subtitle_list, **kwargs) - await self.client.set_category(torrent_hash, "BangumiCollection") + if collection_complete: + if rename_method not in ("none", "normal"): + await self._mark_renamed(torrent_hash, info.get("tags")) + await self.client.set_category(torrent_hash, "BangumiCollection") else: logger.warning(f"{torrent_name} has no media file") + async with Database() as db: + await db.rename_operation.prune_done( + datetime.now(timezone.utc) - timedelta(days=30) + ) logger.debug("Rename process finished.") return renamed_info diff --git a/backend/src/module/manager/revision_policy.py b/backend/src/module/manager/revision_policy.py new file mode 100644 index 00000000..c30c1086 --- /dev/null +++ b/backend/src/module/manager/revision_policy.py @@ -0,0 +1,105 @@ +"""Admission helpers for destructive higher-revision replacement (#1078).""" + +from __future__ import annotations + +import re +import unicodedata +from dataclasses import dataclass +from pathlib import PurePosixPath + +from module.parser.analyser.selector import parse_configured_release_title +from module.parser.analyser.tokenizer import MediaType +from module.parser.release_policy import preference_identity, preference_revision + + +@dataclass(frozen=True, slots=True) +class RevisionIdentity: + bangumi_id: int + media_type: MediaType + season: int + episode: int | float + group: str + resolution: str + revision: int + + +def _normalize_label(value: str | None) -> str: + if not value: + return "" + normalized = unicodedata.normalize("NFKC", value).casefold() + return re.sub(r"[^\w]+", "", normalized) + + +def _adjust_episode(episode: int | float, offset: int) -> int | float: + if episode == 0 and offset: + return 0 + adjusted = episode + offset + if adjusted < 0 or (adjusted == 0 and episode > 0): + return episode + return adjusted + + +def parse_revision_identity( + torrent_name: str, + *, + bangumi_id: int | None, + default_season: int, + episode_offset: int = 0, +) -> RevisionIdentity | None: + """Return the strict identity required before an old file may be deleted.""" + + if bangumi_id is None: + return None + release = parse_configured_release_title(torrent_name) + if release is None: + return None + identity = preference_identity(release, default_season=default_season) + if identity is None: + return None + media_type, season, episode = identity + group = _normalize_label(release.group) + resolution = _normalize_label(release.resolution) + if not group or not resolution: + return None + return RevisionIdentity( + bangumi_id=bangumi_id, + media_type=media_type, + season=season, + episode=_adjust_episode(episode, episode_offset), + group=group, + resolution=resolution, + revision=preference_revision(release), + ) + + +def is_strict_upgrade(old: RevisionIdentity, new: RevisionIdentity) -> bool: + """Whether *new* may destructively replace *old*.""" + + return bool(same_release_identity(old, new) and new.revision > old.revision) + + +def same_release_identity(old: RevisionIdentity, new: RevisionIdentity) -> bool: + """Whether two revisions describe the same strictly matched resource.""" + + return bool( + old.bangumi_id == new.bangumi_id + and old.media_type is new.media_type + and old.season == new.season + and old.episode == new.episode + and old.group == new.group + and old.resolution == new.resolution + ) + + +def replacement_staged_path( + target_path: str, *, old_task_id: str, old_revision: int +) -> str: + """Build a deterministic same-directory temporary path for saga recovery.""" + + normalized = target_path.replace("\\", "/") + path = PurePosixPath(normalized) + suffix = path.suffix + stem = path.name[: -len(suffix)] if suffix else path.name + staged_name = f"{stem}.ab-replaced-v{old_revision}-{old_task_id[:8]}{suffix}" + parent = str(path.parent) + return staged_name if parent == "." else f"{parent}/{staged_name}" diff --git a/backend/src/module/models/__init__.py b/backend/src/module/models/__init__.py index 8b6b8958..6d442aba 100644 --- a/backend/src/module/models/__init__.py +++ b/backend/src/module/models/__init__.py @@ -5,6 +5,11 @@ from .inbox import InboxMessage from .llm_credential import LLMCredential from .movie import Movie, MovieUpdate from .passkey import Passkey, PasskeyCreate, PasskeyDelete, PasskeyList +from .rename_operation import ( + RENAME_OPERATION_STATES, + RenameOperation, + RenameOperationState, +) from .response import APIResponse, ResponseModel from .rss import RSSItem, RSSUpdate from .torrent import EpisodeFile, SubtitleFile, Torrent, TorrentUpdate diff --git a/backend/src/module/models/aria2.py b/backend/src/module/models/aria2.py index 59a7dfe7..9436c194 100644 --- a/backend/src/module/models/aria2.py +++ b/backend/src/module/models/aria2.py @@ -21,4 +21,5 @@ class Aria2Gid(SQLModel, table=True): category: Optional[str] = None dedup_key: Optional[str] = Field(default=None, index=True) renamed_paths: Optional[str] = None + rename_intent: Optional[str] = None created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) diff --git a/backend/src/module/models/config.py b/backend/src/module/models/config.py index 4eef4ddc..14d3cf77 100644 --- a/backend/src/module/models/config.py +++ b/backend/src/module/models/config.py @@ -71,6 +71,10 @@ class BangumiManage(BaseModel): rename_method: str = Field(default="pn", description="Rename method") group_tag: bool = Field(default=False, description="Enable group tag") remove_bad_torrent: bool = Field(default=False, description="Remove bad torrent") + revision_conflict_policy: Literal["hold", "replace"] = Field( + default="hold", + description="How to handle a higher revision targeting an existing episode", + ) # 关闭后 refresh_rss 不再把未匹配种子入库（孤儿记录）；代价是这些条目 # 每轮会被重新内存匹配（廉价），好处是后补规则能立即接住仍在源里的旧集 track_orphans: bool = Field( diff --git a/backend/src/module/models/rename_operation.py b/backend/src/module/models/rename_operation.py new file mode 100644 index 00000000..8428ae76 --- /dev/null +++ b/backend/src/module/models/rename_operation.py @@ -0,0 +1,122 @@ +"""Durable rename/revision-replacement operation state (#1078). + +Downloader calls deliberately happen outside database transactions. This row +is the small, durable state machine that lets a later rename pass reconcile the +observable downloader state and continue an idempotent operation after a retry +or process restart. +""" + +from datetime import datetime, timezone +from typing import Literal + +from sqlalchemy import CheckConstraint, Index, text +from sqlmodel import Field, SQLModel + +RenameOperationState = Literal[ + "conflict", + "retry", + "running", + "planned", + "old_staged", + "new_promoted", + "old_removed", + "done", +] + +RENAME_OPERATION_STATES: tuple[str, ...] = ( + "conflict", + "retry", + "running", + "planned", + "old_staged", + "new_promoted", + "old_removed", + "done", +) + + +def utc_now() -> datetime: + return datetime.now(timezone.utc) + + +class RenameOperation(SQLModel, table=True): + """One conflict or recoverable revision-replacement operation. + + The full identity prevents duplicate rows for one downloader task/path + request. A partial unique index separately reserves an active canonical + target; completed history is excluded so a later V3 operation is not + blocked by an older V2 row. + """ + + __tablename__ = "rename_operation" + __table_args__ = ( + CheckConstraint( + "state IN ('conflict', 'retry', 'running', 'planned', 'old_staged', " + "'new_promoted', 'old_removed', 'done')", + name="ck_rename_operation_state", + ), + CheckConstraint( + "attempt_count >= 0", + name="ck_rename_operation_attempt_count", + ), + Index( + "ux_rename_operation_identity", + "downloader_type", + "new_task_id", + "save_path", + "source_path", + "target_path", + unique=True, + ), + Index( + "ux_rename_operation_active_target", + "downloader_type", + "save_path", + "target_path", + unique=True, + sqlite_where=text("state NOT IN ('done')"), + postgresql_where=text("state NOT IN ('done')"), + ), + Index("ix_rename_operation_state_retry_at", "state", "retry_at"), + Index("ix_rename_operation_new_task_id", "new_task_id"), + Index("ix_rename_operation_old_task_id", "old_task_id"), + ) + + id: int | None = Field(default=None, primary_key=True) + downloader_type: str = Field(max_length=32) + kind: str = Field(default="conflict", max_length=32) + state: str = Field(default="planned", max_length=32) + + # Downloader task ownership. A hold conflict may not have a resolved old + # owner yet; every operation always has the incoming/new task id. + new_task_id: str + old_task_id: str | None = None + + # All paths are downloader-relative except save_path, which is the + # normalized task root. staged_path is deterministic for crash recovery. + save_path: str + source_path: str + target_path: str + staged_path: str | None = None + + # Parsed content identity and revision snapshot used to revalidate a + # replacement after restart. revision_metadata stores the lossless JSON + # snapshot/raw titles for fields that do not warrant schema columns. + bangumi_id: int | None = None + media_type: str | None = Field(default=None, max_length=32) + season: int | None = None + episode: float | None = None + group_name: str | None = None + resolution: str | None = Field(default=None, max_length=32) + old_revision: int | None = None + new_revision: int | None = None + revision_metadata: str | None = None + + attempt_count: int = 0 + retry_at: datetime | None = None + lease_owner: str | None = Field(default=None, max_length=64) + lease_expires_at: datetime | None = None + notified_at: datetime | None = None + last_error: str | None = None + created_at: datetime = Field(default_factory=utc_now) + updated_at: datetime = Field(default_factory=utc_now) diff --git a/backend/src/module/notification/__init__.py b/backend/src/module/notification/__init__.py index b49a7832..78b8fc53 100644 --- a/backend/src/module/notification/__init__.py +++ b/backend/src/module/notification/__init__.py @@ -5,6 +5,7 @@ from .events import ( LLMAuthFailureEvent, LLMPluginInstallFailedEvent, OffsetReviewEvent, + RenameConflictEvent, RssFailureEvent, SystemEvent, UpdateAppliedEvent, @@ -21,6 +22,7 @@ __all__ = [ "NotificationManager", "NotificationProvider", "OffsetReviewEvent", + "RenameConflictEvent", "PROVIDER_REGISTRY", "RssFailureEvent", "SystemEvent", diff --git a/backend/src/module/notification/events.py b/backend/src/module/notification/events.py index 22e84680..0e391f49 100644 --- a/backend/src/module/notification/events.py +++ b/backend/src/module/notification/events.py @@ -248,6 +248,38 @@ class LLMPluginInstallFailedEvent: ) +@dataclass(slots=True, frozen=True) +class RenameConflictEvent: + """A media rename reached a durable target-path conflict.""" + + kind: ClassVar[str] = "rename_conflict" + severity: ClassVar[str] = "warning" + once: ClassVar[bool] = False + + task_id: str + torrent_name: str + target_path: str + reason: str + + def dedup_key(self) -> Optional[str]: + return f"rename_conflict:{self.task_id}:{self.target_path}" + + def payload(self) -> dict: + return { + "task_id": self.task_id, + "torrent_name": self.torrent_name, + "target_path": self.target_path, + "reason": self.reason, + } + + def describe(self) -> tuple[str, str]: + return ( + "媒体文件重命名冲突", + f"种子：{self.torrent_name}\n目标：{self.target_path}\n" + f"原因：{self.reason}", + ) + + # 除「新集数」以外的通知事件的联合类型。 SystemEvent = ( RssFailureEvent @@ -258,4 +290,5 @@ SystemEvent = ( | UpdateAppliedEvent | LLMAuthFailureEvent | LLMPluginInstallFailedEvent + | RenameConflictEvent ) diff --git a/backend/src/test/conftest.py b/backend/src/test/conftest.py index d958cdbb..a9c6c1bd 100644 --- a/backend/src/test/conftest.py +++ b/backend/src/test/conftest.py @@ -125,15 +125,16 @@ def mock_settings(test_settings): @pytest.fixture def mock_qb_client(): """Mock QbDownloader that simulates qBittorrent API responses.""" - from module.downloader import AddResult + from module.downloader import AddResult, RenameOutcome, RenameResult client = AsyncMock() client.auth.return_value = True client.logout.return_value = None client.check_host.return_value = True client.torrents_info.return_value = [] + client.torrent_exists.return_value = False client.torrents_files.return_value = [] - client.torrents_rename_file.return_value = True + client.torrents_rename_file.return_value = RenameResult(RenameOutcome.RENAMED) client.add_torrents.return_value = AddResult.ADDED client.torrents_delete.return_value = None client.torrents_pause.return_value = None diff --git a/backend/src/test/test_api_downloader.py b/backend/src/test/test_api_downloader.py index f596744e..9fddbb66 100644 --- a/backend/src/test/test_api_downloader.py +++ b/backend/src/test/test_api_downloader.py @@ -7,6 +7,7 @@ from fastapi import FastAPI from fastapi.testclient import TestClient from module.api import v1 +from module.models import RenameOperation from module.security.api import get_current_user # --------------------------------------------------------------------------- @@ -77,6 +78,76 @@ class TestAuthRequired: response = unauthed_client.get("/api/v1/downloader/torrents") assert response.status_code == 401 + +class TestRenameConflicts: + def test_list_conflicts(self, authed_client): + row = RenameOperation( + id=7, + downloader_type="qbittorrent", + kind="conflict", + state="conflict", + new_task_id="new-v2", + save_path="/downloads/Show/Season 1", + source_path="raw-v2.mkv", + target_path="Show S01E01.mkv", + last_error="target exists", + ) + with patch("module.api.downloader.Database") as mock_database: + db = MagicMock() + db.rename_operation.list_conflicts = AsyncMock(return_value=[row]) + mock_database.return_value.__aenter__ = AsyncMock(return_value=db) + mock_database.return_value.__aexit__ = AsyncMock(return_value=False) + response = authed_client.get("/api/v1/downloader/rename-conflicts") + + assert response.status_code == 200 + assert response.json()[0]["id"] == 7 + assert response.json()[0]["state"] == "conflict" + + def test_retry_deletes_terminal_conflict(self, authed_client): + row = RenameOperation( + id=7, + downloader_type="qbittorrent", + kind="conflict", + state="conflict", + new_task_id="new-v2", + save_path="/downloads/Show/Season 1", + source_path="raw-v2.mkv", + target_path="Show S01E01.mkv", + ) + with patch("module.api.downloader.Database") as mock_database: + db = MagicMock() + db.rename_operation.get = AsyncMock(return_value=row) + db.rename_operation.delete = AsyncMock(return_value=True) + mock_database.return_value.__aenter__ = AsyncMock(return_value=db) + mock_database.return_value.__aexit__ = AsyncMock(return_value=False) + response = authed_client.post("/api/v1/downloader/rename-conflicts/7/retry") + + assert response.status_code == 200 + db.rename_operation.delete.assert_awaited_once_with(7) + + def test_retry_rejects_active_replacement(self, authed_client): + row = RenameOperation( + id=7, + downloader_type="qbittorrent", + kind="replacement", + state="new_promoted", + new_task_id="new-v2", + old_task_id="old-v1", + save_path="/downloads/Show/Season 1", + source_path="raw-v2.mkv", + target_path="Show S01E01.mkv", + ) + with patch("module.api.downloader.Database") as mock_database: + db = MagicMock() + db.rename_operation.get = AsyncMock(return_value=row) + mock_database.return_value.__aenter__ = AsyncMock(return_value=db) + mock_database.return_value.__aexit__ = AsyncMock(return_value=False) + response = authed_client.post("/api/v1/downloader/rename-conflicts/7/retry") + + assert response.status_code == 409 + + +class TestOtherAuthRequired: @patch("module.security.api.DEV_AUTH_BYPASS", False) def test_pause_torrents_unauthorized(self, unauthed_client): """POST /downloader/torrents/pause without auth returns 401.""" diff --git a/backend/src/test/test_aria2_downloader.py b/backend/src/test/test_aria2_downloader.py index 8b2b74f5..427c0af0 100644 --- a/backend/src/test/test_aria2_downloader.py +++ b/backend/src/test/test_aria2_downloader.py @@ -15,7 +15,8 @@ import httpx import pytest from module.database import Database -from module.downloader import AddResult +from module.database.aria2 import Aria2RenameIntent +from module.downloader import AddResult, RenameOutcome from module.downloader.client.aria2_downloader import ( Aria2ConnectionError, Aria2Downloader, @@ -27,6 +28,18 @@ def _aria2() -> Aria2Downloader: return Aria2Downloader("http://localhost:6800", "u", "secret-token") +def _rename_intent(old_path: str, new_path: str, source) -> Aria2RenameIntent: + stat_result = source.stat() + return Aria2RenameIntent( + old_path=old_path, + new_path=new_path, + st_dev=stat_result.st_dev, + st_ino=stat_result.st_ino, + st_size=stat_result.st_size, + st_mtime_ns=stat_result.st_mtime_ns, + ) + + def _rpc_response(result=None, error=None): resp = MagicMock() body: dict = {"jsonrpc": "2.0", "id": 1} @@ -648,13 +661,47 @@ class TestTorrentsInfo: assert result[0]["tags"] == "" assert result[0]["category"] == "" - async def test_query_failure_returns_empty_list(self): + async def test_connection_query_failure_is_raised(self): aria2 = _aria2() with patch.object( aria2, "_call", AsyncMock(side_effect=Aria2ConnectionError("down")) ): - result = await aria2.torrents_info(status_filter=None, category=None) - assert result == [] + with pytest.raises(Aria2ConnectionError, match="down"): + await aria2.torrents_info(status_filter=None, category=None) + + async def test_rpc_query_failure_is_raised(self): + aria2 = _aria2() + with patch.object( + aria2, "_call", AsyncMock(side_effect=Aria2RpcError(1, "failed")) + ): + with pytest.raises(Aria2RpcError, match="failed"): + await aria2.torrents_info(status_filter=None, category=None) + + async def test_torrent_exists_returns_true_for_known_gid(self): + aria2 = _aria2() + with patch.object( + aria2, "_call", AsyncMock(return_value={"status": "complete"}) + ) as call: + assert await aria2.torrent_exists("gidA") is True + call.assert_awaited_once_with("tellStatus", ["gidA", ["status"]]) + + async def test_torrent_exists_returns_false_only_for_explicit_not_found(self): + aria2 = _aria2() + with patch.object( + aria2, + "_call", + AsyncMock(side_effect=Aria2RpcError(1, "GID#gidA is not found")), + ): + assert await aria2.torrent_exists("gidA") is False + + @pytest.mark.parametrize( + "error", + [Aria2ConnectionError("down"), Aria2RpcError(1, "Internal error")], + ) + async def test_torrent_exists_returns_none_when_presence_is_unknown(self, error): + aria2 = _aria2() + with patch.object(aria2, "_call", AsyncMock(side_effect=error)): + assert await aria2.torrent_exists("gidA") is None async def test_torrents_info_zero_total_length_progress_is_zero(self): """磁力链在拉元数据阶段 totalLength 是字符串 "0"（truthy）—— @@ -802,6 +849,67 @@ class TestTorrentsInfo: assert await db.aria2.get("payload-gid") is not None +# --------------------------------------------------------------------------- +# aria2_gid durable rename intent repository +# --------------------------------------------------------------------------- + + +class TestRenameIntentDatabase: + async def test_persists_and_reads_typed_rename_intent(self, tmp_path): + source = tmp_path / "old.mkv" + source.write_bytes(b"data") + intent = _rename_intent("old.mkv", "new.mkv", source) + + async with Database() as db: + await db.aria2.set_rename_intent("gidA", intent) + assert await db.aria2.get_rename_intent("gidA") == intent + record = await db.aria2.get("gidA") + + assert record is not None + assert record.rename_intent is not None + + async def test_finalize_updates_mapping_and_clears_intent_atomically( + self, tmp_path + ): + source = tmp_path / "old.mkv" + source.write_bytes(b"data") + intent = _rename_intent("old.mkv", "new.mkv", source) + + async with Database() as db: + await db.aria2.set_rename_intent("gidA", intent) + finalized = await db.aria2.finalize_rename_intent("gidA", intent) + + assert finalized is True + assert await db.aria2.get_renamed_paths("gidA") == {"old.mkv": "new.mkv"} + assert await db.aria2.get_rename_intent("gidA") is None + record = await db.aria2.get("gidA") + assert record is not None + assert record.rename_intent is None + + async def test_finalize_rejects_stale_intent_without_mutating_mapping( + self, tmp_path + ): + source = tmp_path / "old.mkv" + source.write_bytes(b"data") + persisted = _rename_intent("old.mkv", "new.mkv", source) + stale = Aria2RenameIntent( + old_path="old.mkv", + new_path="other.mkv", + st_dev=persisted.st_dev, + st_ino=persisted.st_ino, + st_size=persisted.st_size, + st_mtime_ns=persisted.st_mtime_ns, + ) + + async with Database() as db: + await db.aria2.set_rename_intent("gidA", persisted) + finalized = await db.aria2.finalize_rename_intent("gidA", stale) + + assert finalized is False + assert await db.aria2.get_renamed_paths("gidA") == {} + assert await db.aria2.get_rename_intent("gidA") == persisted + + # --------------------------------------------------------------------------- # torrents_files: relative-path mapping # --------------------------------------------------------------------------- @@ -872,7 +980,7 @@ class TestTorrentsRenameFile: with patch.object(aria2, "_call", AsyncMock(side_effect=fake_call)): result = await aria2.torrents_rename_file("gidA", "old.mkv", "new.mkv") - assert result is True + assert result.outcome is RenameOutcome.RENAMED assert not (tmp_path / "old.mkv").exists() assert (tmp_path / "new.mkv").read_bytes() == b"data" @@ -887,7 +995,8 @@ class TestTorrentsRenameFile: return {"dir": str(tmp_path)} with patch.object(aria2, "_call", AsyncMock(side_effect=fake_call)): - assert await aria2.torrents_rename_file("gidA", "old.mkv", "new.mkv") + result = await aria2.torrents_rename_file("gidA", "old.mkv", "new.mkv") + assert result.outcome is RenameOutcome.RENAMED files = await aria2.torrents_files("gidA") assert files == [{"name": "new.mkv", "size": 4}] @@ -903,8 +1012,10 @@ class TestTorrentsRenameFile: return {"dir": str(tmp_path)} with patch.object(aria2, "_call", AsyncMock(side_effect=fake_call)): - assert await aria2.torrents_rename_file("gidA", "old.mkv", "mid.mkv") - assert await aria2.torrents_rename_file("gidA", "mid.mkv", "new.mkv") + first = await aria2.torrents_rename_file("gidA", "old.mkv", "mid.mkv") + second = await aria2.torrents_rename_file("gidA", "mid.mkv", "new.mkv") + assert first.outcome is RenameOutcome.RENAMED + assert second.outcome is RenameOutcome.RENAMED files = await aria2.torrents_files("gidA") assert files == [{"name": "new.mkv", "size": 4}] @@ -921,7 +1032,7 @@ class TestTorrentsRenameFile: "gidA", "old.mkv", os.path.join("Season 01", "new.mkv") ) - assert result is True + assert result.outcome is RenameOutcome.RENAMED assert (tmp_path / "Season 01" / "new.mkv").exists() async def test_returns_false_when_source_file_missing(self, tmp_path): @@ -933,7 +1044,95 @@ class TestTorrentsRenameFile: with patch.object(aria2, "_call", AsyncMock(side_effect=fake_call)): result = await aria2.torrents_rename_file("gidA", "missing.mkv", "new.mkv") - assert result is False + assert result.outcome is RenameOutcome.RETRYABLE_FAILURE + + async def test_recovers_move_completed_before_sidecar_commit(self, tmp_path): + aria2 = _aria2() + old_file = tmp_path / "old.mkv" + new_file = tmp_path / "new.mkv" + old_file.write_bytes(b"data") + intent = _rename_intent("old.mkv", "new.mkv", old_file) + async with Database() as db: + await db.aria2.set_rename_intent("gidA", intent) + old_file.rename(new_file) + + async def fake_call(method, params=None, timeout=10.0): + if method == "getFiles": + return [{"path": str(tmp_path / "old.mkv"), "length": "4"}] + return {"dir": str(tmp_path)} + + with patch.object(aria2, "_call", AsyncMock(side_effect=fake_call)): + result = await aria2.torrents_rename_file("gidA", "old.mkv", "new.mkv") + files = await aria2.torrents_files("gidA") + + assert result.outcome is RenameOutcome.ALREADY_APPLIED + assert files == [{"name": "new.mkv", "size": 4}] + async with Database() as db: + record = await db.aria2.get("gidA") + assert record is not None + assert record.rename_intent is None + + async def test_missing_source_does_not_claim_unrelated_target_without_intent( + self, tmp_path + ): + aria2 = _aria2() + target = tmp_path / "new.mkv" + target.write_bytes(b"unrelated") + + async def fake_call(method, params=None, timeout=10.0): + return {"dir": str(tmp_path)} + + with patch.object(aria2, "_call", AsyncMock(side_effect=fake_call)): + result = await aria2.torrents_rename_file("gidA", "old.mkv", "new.mkv") + + assert result.outcome is RenameOutcome.DESTINATION_EXISTS + assert target.read_bytes() == b"unrelated" + async with Database() as db: + assert await db.aria2.get_renamed_paths("gidA") == {} + + async def test_crash_recovery_rejects_target_with_mismatched_fingerprint( + self, tmp_path + ): + aria2 = _aria2() + old_file = tmp_path / "old.mkv" + old_file.write_bytes(b"original") + intent = _rename_intent("old.mkv", "new.mkv", old_file) + async with Database() as db: + await db.aria2.set_rename_intent("gidA", intent) + old_file.unlink() + (tmp_path / "new.mkv").write_bytes(b"unrelated") + + async def fake_call(method, params=None, timeout=10.0): + return {"dir": str(tmp_path)} + + with patch.object(aria2, "_call", AsyncMock(side_effect=fake_call)): + result = await aria2.torrents_rename_file("gidA", "old.mkv", "new.mkv") + + assert result.outcome is RenameOutcome.DESTINATION_EXISTS + async with Database() as db: + record = await db.aria2.get("gidA") + assert record is not None + assert record.rename_intent is None + assert await db.aria2.get_renamed_paths("gidA") == {} + + async def test_move_failure_clears_rename_intent(self, tmp_path): + aria2 = _aria2() + (tmp_path / "old.mkv").write_bytes(b"data") + + async def fake_call(method, params=None, timeout=10.0): + return {"dir": str(tmp_path)} + + with ( + patch.object(aria2, "_call", AsyncMock(side_effect=fake_call)), + patch.object(aria2, "_move_file", side_effect=OSError("move failed")), + ): + result = await aria2.torrents_rename_file("gidA", "old.mkv", "new.mkv") + + assert result.outcome is RenameOutcome.RETRYABLE_FAILURE + async with Database() as db: + record = await db.aria2.get("gidA") + assert record is not None + assert record.rename_intent is None async def test_torrents_rename_file_existing_destination_refuses_overwrite( self, tmp_path @@ -949,7 +1148,7 @@ class TestTorrentsRenameFile: with patch.object(aria2, "_call", AsyncMock(side_effect=fake_call)): result = await aria2.torrents_rename_file("gidA", "old.mkv", "new.mkv") - assert result is False + assert result.outcome is RenameOutcome.DESTINATION_EXISTS # 两个文件都原封不动。 assert (tmp_path / "old.mkv").read_bytes() == b"source data" assert (tmp_path / "new.mkv").read_bytes() == b"precious existing data" @@ -960,7 +1159,7 @@ class TestTorrentsRenameFile: aria2, "_call", AsyncMock(side_effect=Aria2ConnectionError("down")) ): result = await aria2.torrents_rename_file("gidA", "old.mkv", "new.mkv") - assert result is False + assert result.outcome is RenameOutcome.RETRYABLE_FAILURE # --------------------------------------------------------------------------- @@ -1015,10 +1214,77 @@ class TestTorrentsDelete: raise Aria2RpcError(1, "Internal error") return None + async with Database() as db: + await db.aria2.upsert( + "gidA", bangumi_id=1, renamed_paths='{"old.mkv": "staged.mkv"}' + ) + with patch.object(aria2, "_call", AsyncMock(side_effect=fake_call)): result = await aria2.torrents_delete("gidA", delete_files=False) assert result is False + async with Database() as db: + record = await db.aria2.get("gidA") + assert record is not None + assert record.renamed_paths == '{"old.mkv": "staged.mkv"}' + + async def test_delete_uses_current_renamed_path_not_reoccupied_original( + self, tmp_path + ): + aria2 = _aria2() + original = tmp_path / "Show S01E01.mkv" + staged = tmp_path / ".Show S01E01.ab-old.mkv" + staged.write_bytes(b"old-v1") + original.write_bytes(b"new-v2") + + async def fake_call(method, params=None, timeout=10.0): + if method == "tellStatus": + return {"dir": str(tmp_path)} + if method == "getFiles": + return [{"path": str(original)}] + if method == "forceRemove": + return "gidA" + return None + + async with Database() as db: + await db.aria2.upsert("gidA", bangumi_id=1) + await db.aria2.set_renamed_path("gidA", original.name, staged.name) + + with patch.object(aria2, "_call", AsyncMock(side_effect=fake_call)): + result = await aria2.torrents_delete("gidA", delete_files=True) + + assert result is True + assert not staged.exists() + assert original.read_bytes() == b"new-v2" + async with Database() as db: + assert await db.aria2.get("gidA") is None + + async def test_file_listing_failure_preserves_sidecar_and_task(self): + aria2 = _aria2() + calls = [] + + async def fake_call(method, params=None, timeout=10.0): + calls.append(method) + if method == "tellStatus": + return {"dir": "/downloads"} + if method == "getFiles": + raise Aria2ConnectionError("down") + if method == "forceRemove": + return "gidA" + return None + + async with Database() as db: + await db.aria2.upsert( + "gidA", bangumi_id=1, renamed_paths='{"old.mkv": "staged.mkv"}' + ) + + with patch.object(aria2, "_call", AsyncMock(side_effect=fake_call)): + result = await aria2.torrents_delete("gidA", delete_files=True) + + assert result is False + assert "forceRemove" not in calls + async with Database() as db: + assert await db.aria2.get("gidA") is not None async def test_not_found_error_is_treated_as_already_gone(self): aria2 = _aria2() @@ -1037,6 +1303,26 @@ class TestTorrentsDelete: assert result is True + async def test_not_found_before_file_listing_clears_stale_sidecar(self): + aria2 = _aria2() + + async def fake_call(method, params=None, timeout=10.0): + if method == "tellStatus": + raise Aria2RpcError(1, "GID#gidA is not found") + raise AssertionError(f"unexpected RPC after not-found: {method}") + + async with Database() as db: + await db.aria2.upsert( + "gidA", bangumi_id=1, renamed_paths='{"old.mkv": "staged.mkv"}' + ) + + with patch.object(aria2, "_call", AsyncMock(side_effect=fake_call)): + result = await aria2.torrents_delete("gidA", delete_files=True) + + assert result is True + async with Database() as db: + assert await db.aria2.get("gidA") is None + async def test_accepts_pipe_joined_hashes(self): aria2 = _aria2() seen_gids = [] diff --git a/backend/src/test/test_config.py b/backend/src/test/test_config.py index 0eaf5414..8b52756f 100644 --- a/backend/src/test/test_config.py +++ b/backend/src/test/test_config.py @@ -68,6 +68,7 @@ class TestConfigDefaults: assert config.bangumi_manage.rename_method == "pn" assert config.bangumi_manage.group_tag is False assert config.bangumi_manage.remove_bad_torrent is False + assert config.bangumi_manage.revision_conflict_policy == "hold" assert config.bangumi_manage.eps_complete is False def test_proxy_defaults(self): @@ -380,6 +381,21 @@ class TestEnvOverrides: assert s.rss_parser.engine == "tokenizer" + def test_revision_conflict_policy_from_env(self, tmp_path): + config_file = tmp_path / "config.json" + + with patch.dict( + os.environ, + {"AB_REVISION_CONFLICT_POLICY": "REPLACE"}, + clear=False, + ): + with patch("module.conf.config.CONFIG_PATH", config_file): + s = Settings.__new__(Settings) + Config.__init__(s) + s.init() + + assert s.bangumi_manage.revision_conflict_policy == "replace" + # --------------------------------------------------------------------------- # Security model @@ -601,6 +617,9 @@ class TestDefaultSettings: """Factory defaults preserve the pre-Preview parser behaviour.""" assert DEFAULT_SETTINGS["rss_parser"]["engine"] == "classic" + def test_revision_conflict_policy_defaults_to_hold(self): + assert DEFAULT_SETTINGS["bangumi_manage"]["revision_conflict_policy"] == "hold" + # --------------------------------------------------------------------------- # BCOLORS utility diff --git a/backend/src/test/test_database_rename_operation.py b/backend/src/test/test_database_rename_operation.py new file mode 100644 index 00000000..e672caef --- /dev/null +++ b/backend/src/test/test_database_rename_operation.py @@ -0,0 +1,270 @@ +"""RenameOperationDatabase — #1078 durable rename/replacement state.""" + +from datetime import datetime, timedelta, timezone + +import pytest +from sqlalchemy.exc import IntegrityError + +from module.database import Database +from module.database.rename_operation import RenameOperationDatabase +from module.models import RenameOperation + + +def _operation(**overrides) -> RenameOperation: + values = { + "downloader_type": "qbittorrent", + "kind": "replacement", + "state": "planned", + "new_task_id": "new-v2", + "old_task_id": "old-v1", + "save_path": "/downloads/Anime/Season 1", + "source_path": "[ANi] Anime - 01 [V2].mp4", + "target_path": "Anime S01E01.mp4", + "staged_path": ".ab-replace/old-v1/Anime S01E01.mp4", + "bangumi_id": 42, + "media_type": "episode", + "season": 1, + "episode": 1.0, + "group_name": "ANi", + "resolution": "1080p", + "old_revision": 1, + "new_revision": 2, + "revision_metadata": '{"old":"V1","new":"V2"}', + } + values.update(overrides) + return RenameOperation(**values) + + +async def test_get_or_create_persists_complete_operation_metadata(db_session): + repo = RenameOperationDatabase(db_session) + + row, created = await repo.get_or_create(_operation()) + + assert created is True + assert row.id is not None + assert row.old_task_id == "old-v1" + assert row.staged_path == ".ab-replace/old-v1/Anime S01E01.mp4" + assert row.old_revision == 1 + assert row.new_revision == 2 + assert row.revision_metadata == '{"old":"V1","new":"V2"}' + assert row.attempt_count == 0 + assert row.created_at is not None + assert row.updated_at is not None + + same, duplicate_created = await repo.get_or_create(_operation()) + assert duplicate_created is False + assert same.id == row.id + assert await repo.get(row.id) == row + + +async def test_active_target_is_unique_but_done_history_does_not_block_v3(db_session): + repo = RenameOperationDatabase(db_session) + first, _ = await repo.get_or_create(_operation(state="conflict")) + + with pytest.raises(IntegrityError): + await repo.get_or_create( + _operation( + new_task_id="new-v3", + source_path="[ANi] Anime - 01 [V3].mp4", + new_revision=3, + ) + ) + + await repo.set_state(first.id, "done") + v3, created = await repo.get_or_create( + _operation( + new_task_id="new-v3", + source_path="[ANi] Anime - 01 [V3].mp4", + new_revision=3, + ) + ) + assert created is True + assert v3.new_revision == 3 + + +async def test_upsert_conflict_is_idempotent_and_refreshes_error(db_session): + repo = RenameOperationDatabase(db_session) + + first, created = await repo.upsert_conflict( + _operation(state="retry", last_error="qB 409") + ) + assert created is True + assert first.state == "conflict" + assert first.retry_at is None + + same, created_again = await repo.upsert_conflict( + _operation( + state="planned", + old_task_id="resolved-old-owner", + last_error="destination still exists", + ) + ) + assert created_again is False + assert same.id == first.id + assert same.state == "conflict" + assert same.old_task_id == "resolved-old-owner" + assert same.last_error == "destination still exists" + + +async def test_claim_is_atomic_due_only_and_increments_attempt(db_session): + repo = RenameOperationDatabase(db_session) + now = datetime.now(timezone.utc) + row, _ = await repo.get_or_create( + _operation(state="retry", retry_at=now - timedelta(seconds=1)) + ) + + claimed = await repo.claim( + row.id, + from_states=("retry",), + to_state="planned", + now=now, + ) + + assert claimed is not None + assert claimed.state == "planned" + assert claimed.attempt_count == 1 + assert claimed.retry_at is None + assert ( + await repo.claim( + row.id, + from_states=("retry",), + to_state="planned", + now=now, + ) + is None + ) + + future, _ = await repo.get_or_create( + _operation( + new_task_id="future-v3", + source_path="future-v3.mkv", + target_path="Anime S01E02.mp4", + state="retry", + retry_at=now + timedelta(hours=1), + ) + ) + assert ( + await repo.claim( + future.id, + from_states=("retry",), + to_state="planned", + now=now, + ) + is None + ) + + +async def test_mark_notified_is_one_shot(db_session): + repo = RenameOperationDatabase(db_session) + row, _ = await repo.upsert_conflict(_operation()) + notified_at = datetime.now(timezone.utc) + + assert await repo.mark_notified(row.id, notified_at) is True + assert await repo.mark_notified(row.id, notified_at + timedelta(seconds=1)) is False + + loaded = await repo.get(row.id) + assert loaded is not None + assert loaded.notified_at == notified_at + + +async def test_recover_stale_running_uses_compare_and_swap(db_session): + repo = RenameOperationDatabase(db_session) + now = datetime.now(timezone.utc) + row, _ = await repo.get_or_create(_operation(state="running")) + row.updated_at = now - timedelta(minutes=10) + db_session.add(row) + await db_session.commit() + + assert await repo.recover_stale_running(row.id, before=now - timedelta(minutes=5)) + assert not await repo.recover_stale_running( + row.id, before=now - timedelta(minutes=5) + ) + loaded = await repo.get(row.id) + assert loaded is not None + assert loaded.state == "retry" + + +async def test_replacement_phase_lease_fences_workers_and_state_commit(db_session): + repo = RenameOperationDatabase(db_session) + now = datetime.now(timezone.utc) + row, _ = await repo.get_or_create(_operation()) + + first = await repo.claim_replacement_lease(row.id, owner="worker-a", now=now) + second = await repo.claim_replacement_lease(row.id, owner="worker-b", now=now) + + assert first is not None + assert first.lease_owner == "worker-a" + assert first.attempt_count == 1 + assert second is None + assert ( + await repo.set_state_claimed(row.id, owner="worker-b", state="old_staged") + is None + ) + + advanced = await repo.set_state_claimed( + row.id, owner="worker-a", state="old_staged" + ) + assert advanced is not None + assert advanced.state == "old_staged" + assert advanced.lease_owner is None + assert advanced.lease_expires_at is None + + +async def test_expired_replacement_phase_lease_can_be_reclaimed(db_session): + repo = RenameOperationDatabase(db_session) + now = datetime.now(timezone.utc) + row, _ = await repo.get_or_create(_operation()) + assert await repo.claim_replacement_lease( + row.id, + owner="crashed-worker", + now=now - timedelta(minutes=10), + lease_for=timedelta(minutes=5), + ) + + reclaimed = await repo.claim_replacement_lease( + row.id, owner="recovery-worker", now=now + ) + + assert reclaimed is not None + assert reclaimed.lease_owner == "recovery-worker" + assert reclaimed.attempt_count == 2 + + +async def test_retry_query_and_done_pruning(db_session): + repo = RenameOperationDatabase(db_session) + now = datetime.now(timezone.utc) + due, _ = await repo.get_or_create( + _operation(state="retry", retry_at=now - timedelta(seconds=1)) + ) + await repo.get_or_create( + _operation( + new_task_id="later-v3", + source_path="later-v3.mkv", + target_path="Anime S01E02.mp4", + state="retry", + retry_at=now + timedelta(hours=1), + ) + ) + conflict, _ = await repo.upsert_conflict( + _operation( + new_task_id="conflict-v4", + source_path="conflict-v4.mkv", + target_path="Anime S01E03.mp4", + ) + ) + + assert [row.id for row in await repo.list_retryable(now)] == [due.id] + assert [row.id for row in await repo.list_conflicts()] == [conflict.id] + + await repo.set_state(conflict.id, "done") + conflict.updated_at = now - timedelta(days=31) + db_session.add(conflict) + await db_session.commit() + assert await repo.prune_done(before=now - timedelta(days=30)) == 1 + + +async def test_database_facade_exposes_rename_operation_repository(db_engine): + async with Database(engine=db_engine) as db: + row, created = await db.rename_operation.get_or_create(_operation()) + assert created is True + assert await db.rename_operation.get(row.id) is not None diff --git a/backend/src/test/test_download_client.py b/backend/src/test/test_download_client.py index 40cfeaf4..9ad07348 100644 --- a/backend/src/test/test_download_client.py +++ b/backend/src/test/test_download_client.py @@ -4,6 +4,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest +from module.downloader.base import RenameOutcome, RenameResult from module.downloader.download_client import ( TORRENT_FETCH_PER_HOST_DELAY, AddResult, @@ -335,27 +336,44 @@ class TestClientDelegation: ) assert len(result) == 1 + async def test_torrent_exists_preserves_unknown_state( + self, download_client, mock_qb_client + ): + mock_qb_client.torrent_exists.return_value = None + + result = await download_client.torrent_exists("hash1") + + assert result is None + mock_qb_client.torrent_exists.assert_awaited_once_with("hash1") + async def test_rename_torrent_file_success(self, download_client, mock_qb_client): - """rename_torrent_file returns True on success.""" - mock_qb_client.torrents_rename_file.return_value = True + """rename_torrent_file preserves the concrete structured result.""" + expected = RenameResult(RenameOutcome.RENAMED) + mock_qb_client.torrents_rename_file.return_value = expected result = await download_client.rename_torrent_file( "hash1", "old.mkv", "new.mkv" ) - assert result is True + assert result is expected async def test_rename_torrent_file_failure(self, download_client, mock_qb_client): - """rename_torrent_file returns False on failure.""" - mock_qb_client.torrents_rename_file.return_value = False + """Failure reasons are not collapsed back into a boolean.""" + expected = RenameResult( + RenameOutcome.RETRYABLE_FAILURE, detail="verification timed out" + ) + mock_qb_client.torrents_rename_file.return_value = expected result = await download_client.rename_torrent_file( "hash1", "old.mkv", "new.mkv" ) - assert result is False + assert result is expected + assert result.outcome is RenameOutcome.RETRYABLE_FAILURE async def test_rename_torrent_file_passes_verify_flag( self, download_client, mock_qb_client ): """rename_torrent_file forwards the verify kwarg to the underlying client.""" - mock_qb_client.torrents_rename_file.return_value = True + mock_qb_client.torrents_rename_file.return_value = RenameResult( + RenameOutcome.RENAMED + ) await download_client.rename_torrent_file( "hash1", "old.mkv", "new.mkv", verify=False ) diff --git a/backend/src/test/test_downloader_protocol.py b/backend/src/test/test_downloader_protocol.py index d767270a..232923d1 100644 --- a/backend/src/test/test_downloader_protocol.py +++ b/backend/src/test/test_downloader_protocol.py @@ -9,6 +9,8 @@ from module.downloader.base import ( CoreDownloaderClient, DownloaderCapabilities, DownloaderClient, + RenameOutcome, + RenameResult, ) from module.downloader.client.aria2_downloader import Aria2Downloader from module.downloader.client.mock_downloader import MockDownloader @@ -48,6 +50,16 @@ class TestCapabilities: assert isinstance(cls.capabilities, DownloaderCapabilities) +class TestRenameResult: + def test_success_outcomes_preserve_boolean_compatibility(self): + assert RenameResult(RenameOutcome.RENAMED) + assert RenameResult(RenameOutcome.ALREADY_APPLIED) + + def test_failure_outcomes_are_false(self): + assert not RenameResult(RenameOutcome.DESTINATION_EXISTS) + assert not RenameResult(RenameOutcome.RETRYABLE_FAILURE) + + class TestProtocolConformance: def test_qb_satisfies_full_protocol(self): assert isinstance(_qb(), DownloaderClient) diff --git a/backend/src/test/test_loops.py b/backend/src/test/test_loops.py index 60f36b86..0473e51e 100644 --- a/backend/src/test/test_loops.py +++ b/backend/src/test/test_loops.py @@ -21,6 +21,7 @@ from module.core.loops import ( from module.models.bangumi import Notification from module.notification.events import ( OffsetReviewEvent, + RenameConflictEvent, RssFailureEvent, UpdateAvailableEvent, ) @@ -144,6 +145,32 @@ class TestRssTick: class TestRenameTick: + async def test_persists_and_dispatches_conflict_event_when_push_disabled(self): + event = RenameConflictEvent( + task_id="new-v2", + torrent_name="Episode V2", + target_path="Show S01E01.mkv", + reason="target exists", + ) + notifier = AsyncMock() + mock_renamer = AsyncMock() + mock_renamer.rename = AsyncMock(return_value=[]) + mock_renamer.events = [event] + + with ( + patch( + "module.core.loops.DownloadClient", + return_value=_async_cm(AsyncMock()), + ), + patch("module.core.loops.Renamer", return_value=mock_renamer), + patch("module.core.loops.settings") as mock_settings, + ): + mock_settings.notification.enable = False + await rename_tick(notifier) + + notifier.send_event.assert_awaited_once_with(event) + notifier.send_all.assert_not_awaited() + async def test_sends_notification_per_renamed_item(self): """Every item Renamer.rename() returns is forwarded to the notifier.""" notify = Notification(official_title="Test Anime", season=1, episode=1) diff --git a/backend/src/test/test_migrations_module.py b/backend/src/test/test_migrations_module.py index c5283cfe..893e2317 100644 --- a/backend/src/test/test_migrations_module.py +++ b/backend/src/test/test_migrations_module.py @@ -375,6 +375,14 @@ class TestRunMigrations: aria2_cols = {c["name"] for c in inspector.get_columns("aria2_gid")} assert "renamed_paths" in aria2_cols + def test_v24_adds_aria2_rename_intent_column(self): + """v24 persists an ownership proof before moving an aria2 file.""" + engine = _make_v0_engine() + run_migrations(engine) + + aria2_cols = _columns(engine, "aria2_gid") + assert "rename_intent" in aria2_cols + def test_creates_inboxmessage_table(self): """v16 creates the in-app notification center table with its indexes.""" engine = _make_v0_engine() @@ -399,6 +407,110 @@ class TestRunMigrations: indexes = {ix["name"]: ix for ix in inspector.get_indexes("llmcredential")} assert indexes["ix_llmcredential_provider_id"]["unique"] + def test_v23_creates_durable_rename_operation_schema(self): + engine = _make_v0_engine() + + run_migrations(engine) + + inspector = inspect(engine) + assert "rename_operation" in inspector.get_table_names() + assert { + "downloader_type", + "kind", + "state", + "new_task_id", + "old_task_id", + "save_path", + "source_path", + "target_path", + "staged_path", + "bangumi_id", + "media_type", + "season", + "episode", + "group_name", + "resolution", + "old_revision", + "new_revision", + "revision_metadata", + "attempt_count", + "retry_at", + "notified_at", + "last_error", + "created_at", + "updated_at", + } <= _columns(engine, "rename_operation") + indexes = { + item["name"]: item for item in inspector.get_indexes("rename_operation") + } + assert { + "ux_rename_operation_identity", + "ux_rename_operation_active_target", + "ix_rename_operation_state_retry_at", + "ix_rename_operation_new_task_id", + "ix_rename_operation_old_task_id", + } <= set(indexes) + assert indexes["ux_rename_operation_identity"]["unique"] + assert indexes["ux_rename_operation_active_target"]["unique"] + + def test_v23_active_target_unique_index_ignores_done_history(self): + engine = _make_v0_engine() + run_migrations(engine) + insert = text( + "INSERT INTO rename_operation " + "(downloader_type, kind, state, new_task_id, save_path, " + " source_path, target_path, attempt_count, created_at, updated_at) " + "VALUES ('qbittorrent', 'replacement', :state, :task, '/show', " + " :source, 'Show S01E01.mkv', 0, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)" + ) + with engine.begin() as conn: + conn.execute(insert, {"state": "conflict", "task": "v2", "source": "v2"}) + with pytest.raises(IntegrityError): + with engine.begin() as conn: + conn.execute( + insert, + {"state": "planned", "task": "v3", "source": "v3"}, + ) + with engine.begin() as conn: + conn.execute( + text( + "UPDATE rename_operation SET state = 'done' WHERE new_task_id = 'v2'" + ) + ) + conn.execute( + insert, + {"state": "planned", "task": "v3", "source": "v3"}, + ) + + def test_v23_rejects_unknown_operation_state(self): + engine = _make_v0_engine() + run_migrations(engine) + + with pytest.raises(IntegrityError): + with engine.begin() as conn: + conn.execute( + text( + "INSERT INTO rename_operation " + "(downloader_type, kind, state, new_task_id, save_path, " + " source_path, target_path, attempt_count, created_at, updated_at) " + "VALUES ('qbittorrent', 'conflict', 'unknown', 'v2', '/show', " + " 'v2.mkv', 'Show S01E01.mkv', 0, CURRENT_TIMESTAMP, " + " CURRENT_TIMESTAMP)" + ) + ) + + def test_v23_guard_accepts_metadata_created_table(self, monkeypatch): + engine = _make_v0_engine() + _run_through_version(engine, 22, monkeypatch) + SQLModel.metadata.create_all(engine) + + run_migrations(engine) + + with engine.connect() as conn: + assert get_schema_version(conn) == CURRENT_SCHEMA_VERSION + indexes = {ix["name"] for ix in inspect(engine).get_indexes("rename_operation")} + assert "ux_rename_operation_active_target" in indexes + def test_upgrades_auth_beta_v20_without_losing_tokens(self, monkeypatch): engine = _make_v19_auth_engine() _run_through_version(engine, 20, monkeypatch) diff --git a/backend/src/test/test_mock_downloader.py b/backend/src/test/test_mock_downloader.py index 5e129af5..65cf684c 100644 --- a/backend/src/test/test_mock_downloader.py +++ b/backend/src/test/test_mock_downloader.py @@ -2,7 +2,7 @@ import pytest -from module.downloader import AddResult +from module.downloader import AddResult, RenameOutcome from module.downloader.client.mock_downloader import MockDownloader @@ -192,6 +192,12 @@ class TestMockDownloaderTorrentsInfo: result = await mock_dl.torrents_info(status_filter=None, category="Bangumi") assert result == [] + async def test_torrent_exists_distinguishes_present_and_absent(self, mock_dl): + torrent_hash = mock_dl.add_mock_torrent("Anime A", category="Bangumi") + + assert await mock_dl.torrent_exists(torrent_hash) is True + assert await mock_dl.torrent_exists("missing") is False + class TestMockDownloaderTorrentsFiles: async def test_returns_files_for_known_hash(self, mock_dl): @@ -257,7 +263,7 @@ class TestMockDownloaderRename: old_path="old.mkv", new_path="new.mkv", ) - assert result is True + assert result.outcome is RenameOutcome.RENAMED async def test_rename_with_verify_flag(self, mock_dl): result = await mock_dl.torrents_rename_file( @@ -266,7 +272,7 @@ class TestMockDownloaderRename: new_path="new.mkv", verify=False, ) - assert result is True + assert result.outcome is RenameOutcome.RENAMED # --------------------------------------------------------------------------- diff --git a/backend/src/test/test_qb_downloader.py b/backend/src/test/test_qb_downloader.py index 01e2538b..b78e7cf0 100644 --- a/backend/src/test/test_qb_downloader.py +++ b/backend/src/test/test_qb_downloader.py @@ -12,7 +12,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest -from module.downloader import AddResult +from module.downloader import AddResult, RenameOutcome from module.downloader.client.qb_downloader import QbDownloader # --------------------------------------------------------------------------- @@ -1181,6 +1181,34 @@ class TestTorrentsInfoPausedFilter: assert sent_params["filter"] == "completed" +class TestTorrentExists: + async def test_queries_exact_hash_and_confirms_presence(self): + qb = QbDownloader(host="localhost:8080", username="u", password="p", ssl=False) + qb._client = AsyncMock() + response = MagicMock(status_code=200) + response.json.return_value = [{"hash": "ABC123"}] + qb._client.request = AsyncMock(return_value=response) + + assert await qb.torrent_exists("abc123") is True + assert qb._client.request.call_args.kwargs["params"] == {"hashes": "abc123"} + + async def test_empty_exact_hash_query_confirms_absence(self): + qb = QbDownloader(host="localhost:8080", username="u", password="p", ssl=False) + qb._client = AsyncMock() + response = MagicMock(status_code=200) + response.json.return_value = [] + qb._client.request = AsyncMock(return_value=response) + + assert await qb.torrent_exists("missing") is False + + async def test_failed_query_is_unknown_not_absent(self): + qb = QbDownloader(host="localhost:8080", username="u", password="p", ssl=False) + qb._client = AsyncMock() + qb._client.request = AsyncMock(side_effect=ConnectionError("offline")) + + assert await qb.torrent_exists("abc123") is None + + # --------------------------------------------------------------------------- # torrents_delete (#1046) # --------------------------------------------------------------------------- @@ -1250,26 +1278,42 @@ class TestRenameVerifyContract: return qb async def test_verify_false_when_file_keeps_old_name(self): - """API 200 but the file never gets renamed -> False.""" + """API 200 but the file never gets renamed -> retryable.""" qb = self._make_qb([{"name": "old.mkv"}]) - assert await qb.torrents_rename_file("h", "old.mkv", "new.mkv") is False + result = await qb.torrents_rename_file("h", "old.mkv", "new.mkv") + assert result.outcome is RenameOutcome.RETRYABLE_FAILURE async def test_verify_false_when_neither_name_present(self): - """Neither old nor new name in the file list -> False, not a blind True.""" + """Neither old nor new name is retryable, not a blind success.""" qb = self._make_qb([{"name": "unrelated.mkv"}]) - assert await qb.torrents_rename_file("h", "old.mkv", "new.mkv") is False + result = await qb.torrents_rename_file("h", "old.mkv", "new.mkv") + assert result.outcome is RenameOutcome.RETRYABLE_FAILURE async def test_verify_true_when_new_path_appears(self): """A real, verified rename returns True.""" qb = self._make_qb([{"name": "new.mkv"}]) - assert await qb.torrents_rename_file("h", "old.mkv", "new.mkv") is True + result = await qb.torrents_rename_file("h", "old.mkv", "new.mkv") + assert result.outcome is RenameOutcome.RENAMED async def test_verify_409_conflict_is_false(self): - """Target already exists (duplicate from another source) -> False.""" + """Target already exists is a terminal, distinguishable outcome.""" qb = self._make_qb([{"name": "old.mkv"}], post_code=409) - assert await qb.torrents_rename_file("h", "old.mkv", "new.mkv") is False + result = await qb.torrents_rename_file("h", "old.mkv", "new.mkv") + assert result.outcome is RenameOutcome.DESTINATION_EXISTS async def test_rename_204_with_verified_new_path_is_true(self): """qB 5.2 对成功的 renameFile 回 204；不能再用 ==200 判定失败。""" qb = self._make_qb([{"name": "new.mkv"}], post_code=204) - assert await qb.torrents_rename_file("h", "old.mkv", "new.mkv") is True + result = await qb.torrents_rename_file("h", "old.mkv", "new.mkv") + assert result.outcome is RenameOutcome.RENAMED + + async def test_http_error_is_retryable(self): + qb = self._make_qb([{"name": "old.mkv"}], post_code=500) + result = await qb.torrents_rename_file("h", "old.mkv", "new.mkv") + assert result.outcome is RenameOutcome.RETRYABLE_FAILURE + + async def test_network_error_is_retryable(self): + qb = self._make_qb([]) + qb._post = AsyncMock(side_effect=httpx.ConnectError("down")) + result = await qb.torrents_rename_file("h", "old.mkv", "new.mkv") + assert result.outcome is RenameOutcome.RETRYABLE_FAILURE diff --git a/backend/src/test/test_renamer.py b/backend/src/test/test_renamer.py index 32e5af57..725e8ddd 100644 --- a/backend/src/test/test_renamer.py +++ b/backend/src/test/test_renamer.py @@ -5,8 +5,8 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest from module.conf import settings -from module.downloader import DownloadClient -from module.manager.renamer import Renamer +from module.downloader import DownloadClient, RenameOutcome, RenameResult +from module.manager.renamer import PreparedMediaRename, Renamer from module.models import EpisodeFile, Notification, SubtitleFile # --------------------------------------------------------------------------- @@ -996,6 +996,519 @@ class TestRenameFlow: renamer.client.client.torrents_rename_file.assert_not_called() +class TestRevisionConflictFlow: + V1 = "[ANi] 尼古喵喵 - 01 [1080P][Baha][WEB-DL][AAC AVC][CHT].mp4" + V2 = "[ANi] 尼古喵喵 - 01 [V2][1080P][Baha][WEB-DL][AAC AVC][CHT].mp4" + TARGET = "尼古喵喵 S01E01.mp4" + SAVE_PATH = "/downloads/Bangumi/尼古喵喵 (2026)/Season 1" + + @pytest.fixture + def renamer(self, mock_qb_client): + with patch("module.downloader.download_client.settings") as mock_settings: + mock_settings.downloader.type = "qbittorrent" + mock_settings.downloader.host = "localhost:8080" + mock_settings.downloader.username = "admin" + mock_settings.downloader.password = "admin" + mock_settings.downloader.ssl = False + mock_settings.downloader.path = "/downloads/Bangumi" + mock_settings.bangumi_manage.group_tag = False + mock_settings.bangumi_manage.remove_bad_torrent = False + with patch( + "module.downloader.download_client.DownloadClient._DownloadClient__getClient", + return_value=mock_qb_client, + ): + client = DownloadClient() + client.client = mock_qb_client + + async def torrent_exists(torrent_hash): + infos = mock_qb_client.torrents_info.return_value + return isinstance(infos, list) and any( + item.get("hash") == torrent_hash for item in infos + ) + + mock_qb_client.torrent_exists.side_effect = torrent_exists + return Renamer(client) + + def _infos(self): + return [ + { + "hash": "old-v1", + "name": self.V1, + "save_path": self.SAVE_PATH, + "tags": "ab:42, ab:renamed", + }, + { + "hash": "new-v2", + "name": self.V2, + "save_path": self.SAVE_PATH, + "tags": "ab:42", + }, + ] + + @staticmethod + def _offsets(): + return {"new-v2": (0, 0, "episode")} + + async def test_renamed_tag_skips_before_file_and_offset_queries(self, renamer): + renamer.client.client.torrents_info.return_value = [self._infos()[0]] + with patch.object(renamer, "_batch_lookup_offsets", AsyncMock()) as lookup: + assert await renamer.rename() == [] + + renamer.client.client.torrents_files.assert_not_awaited() + lookup.assert_not_awaited() + + async def test_overlapping_ordinary_rename_is_claimed_once( + self, renamer, test_settings + ): + import asyncio + + info = self._infos()[1] + renamer.client.client.torrents_info.return_value = [info] + renamer.client.client.torrents_files.return_value = [{"name": self.V2}] + renamer.client.client.torrents_rename_file.return_value = RenameResult( + RenameOutcome.RENAMED + ) + test_settings.bangumi_manage.rename_method = "pn" + other = Renamer(renamer.client) + + with ( + patch("module.manager.renamer.settings", test_settings), + patch.object( + renamer, + "_batch_lookup_offsets", + AsyncMock(return_value=self._offsets()), + ), + patch.object( + other, + "_batch_lookup_offsets", + AsyncMock(return_value=self._offsets()), + ), + ): + results = await asyncio.gather(renamer.rename(), other.rename()) + + assert sum(len(result) for result in results) == 1 + renamer.client.client.torrents_rename_file.assert_awaited_once() + + from module.database import Database + + async with Database() as db: + operation = await db.rename_operation.get_by_target( + downloader_type=renamer._downloader_type(), + save_path=self.SAVE_PATH, + target_path=self.TARGET, + active_only=False, + ) + assert operation is not None + assert operation.state == "done" + assert operation.attempt_count == 1 + + async def test_ordinary_rename_reconciles_downloader_success_after_crash( + self, renamer + ): + info = self._infos()[1] + # qB already exposes the canonical name, but the durable operation row + # was not marked done before the previous process stopped. + renamer.client.client.torrents_files.return_value = [{"name": self.TARGET}] + prepared = PreparedMediaRename( + episode=EpisodeFile( + media_path=self.V2, + title="尼古喵喵", + season=1, + episode=1, + suffix=".mp4", + ), + source_path=self.V2, + target_path=self.TARGET, + ) + + report = await renamer._run_ordinary_rename( + info=info, + prepared=prepared, + identity=None, + bangumi_name="尼古喵喵", + episode_offset=0, + ) + + assert report.result.outcome is RenameOutcome.ALREADY_APPLIED + renamer.client.client.torrents_rename_file.assert_not_awaited() + from module.database import Database + + async with Database() as db: + operation = await db.rename_operation.get_by_target( + downloader_type=renamer._downloader_type(), + save_path=self.SAVE_PATH, + target_path=self.TARGET, + active_only=False, + ) + assert operation is not None + assert operation.state == "done" + + async def test_hold_conflict_is_persistent_and_notified_once( + self, renamer, test_settings + ): + renamer.client.client.torrents_info.return_value = self._infos() + + async def files(torrent_hash): + if torrent_hash == "old-v1": + return [{"name": self.TARGET}] + return [{"name": self.V2}] + + renamer.client.client.torrents_files.side_effect = files + test_settings.bangumi_manage.rename_method = "pn" + test_settings.bangumi_manage.revision_conflict_policy = "hold" + + with ( + patch("module.manager.renamer.settings", test_settings), + patch.object( + renamer, + "_batch_lookup_offsets", + AsyncMock(return_value=self._offsets()), + ), + ): + assert await renamer.rename() == [] + + assert len(renamer.events) == 1 + renamer.client.client.torrents_rename_file.assert_not_awaited() + + restarted = Renamer(renamer.client) + with ( + patch("module.manager.renamer.settings", test_settings), + patch.object( + restarted, + "_batch_lookup_offsets", + AsyncMock(return_value=self._offsets()), + ), + ): + assert await restarted.rename() == [] + + assert restarted.events == [] + renamer.client.client.torrents_rename_file.assert_not_awaited() + + async def test_replace_stages_promotes_then_deletes_old( + self, renamer, test_settings + ): + infos = self._infos() + renamer.client.client.torrents_info.return_value = infos + paths = {"old-v1": self.TARGET, "new-v2": self.V2} + + async def files(torrent_hash): + return [{"name": paths[torrent_hash]}] + + async def rename(torrent_hash, old_path, new_path, verify=True): + assert paths[torrent_hash] == old_path + paths[torrent_hash] = new_path + return RenameResult(RenameOutcome.RENAMED) + + async def delete(torrent_hash, delete_files=True): + infos[:] = [info for info in infos if info["hash"] != torrent_hash] + return True + + renamer.client.client.torrents_files.side_effect = files + renamer.client.client.torrents_rename_file.side_effect = rename + renamer.client.client.torrents_delete.side_effect = delete + test_settings.bangumi_manage.rename_method = "pn" + test_settings.bangumi_manage.revision_conflict_policy = "replace" + + with ( + patch("module.manager.renamer.settings", test_settings), + patch.object( + renamer, + "_batch_lookup_offsets", + AsyncMock(return_value=self._offsets()), + ), + ): + result = await renamer.rename() + + assert len(result) == 1 + assert paths["new-v2"] == self.TARGET + assert ".ab-replaced-v1-old-v1" in paths["old-v1"] + renamer.client.client.torrents_delete.assert_awaited_once_with( + "old-v1", delete_files=True + ) + assert renamer.events == [] + + async def test_promotion_failure_restores_v1_and_waits_for_retry( + self, renamer, test_settings + ): + renamer.client.client.torrents_info.return_value = self._infos() + paths = {"old-v1": self.TARGET, "new-v2": self.V2} + + async def files(torrent_hash): + return [{"name": paths[torrent_hash]}] + + async def rename(torrent_hash, old_path, new_path, verify=True): + if torrent_hash == "new-v2": + return RenameResult( + RenameOutcome.RETRYABLE_FAILURE, detail="qB unavailable" + ) + assert paths[torrent_hash] == old_path + paths[torrent_hash] = new_path + return RenameResult(RenameOutcome.RENAMED) + + renamer.client.client.torrents_files.side_effect = files + renamer.client.client.torrents_rename_file.side_effect = rename + renamer.client.client.torrents_delete.return_value = True + test_settings.bangumi_manage.rename_method = "pn" + test_settings.bangumi_manage.revision_conflict_policy = "replace" + + with ( + patch("module.manager.renamer.settings", test_settings), + patch.object( + renamer, + "_batch_lookup_offsets", + AsyncMock(return_value=self._offsets()), + ), + ): + assert await renamer.rename() == [] + + assert paths["old-v1"] == self.TARGET + assert paths["new-v2"] == self.V2 + assert renamer.client.client.torrents_rename_file.await_count == 3 + renamer.client.client.torrents_delete.assert_not_awaited() + + restarted = Renamer(renamer.client) + with ( + patch("module.manager.renamer.settings", test_settings), + patch.object( + restarted, + "_batch_lookup_offsets", + AsyncMock(return_value=self._offsets()), + ), + ): + assert await restarted.rename() == [] + assert renamer.client.client.torrents_rename_file.await_count == 3 + + async def test_multifile_candidate_falls_back_to_hold(self, renamer, test_settings): + renamer.client.client.torrents_info.return_value = self._infos() + + async def files(torrent_hash): + if torrent_hash == "old-v1": + return [{"name": self.TARGET}] + return [{"name": self.V2}, {"name": "尼古喵喵 - 01.ass"}] + + renamer.client.client.torrents_files.side_effect = files + test_settings.bangumi_manage.rename_method = "pn" + test_settings.bangumi_manage.revision_conflict_policy = "replace" + + with ( + patch("module.manager.renamer.settings", test_settings), + patch.object( + renamer, + "_batch_lookup_offsets", + AsyncMock(return_value=self._offsets()), + ), + ): + assert await renamer.rename() == [] + + assert len(renamer.events) == 1 + assert "single-file" in renamer.events[0].reason + renamer.client.client.torrents_rename_file.assert_not_awaited() + renamer.client.client.torrents_delete.assert_not_awaited() + + async def test_missing_v2_after_staging_restores_v1_before_conflict( + self, renamer, test_settings + ): + renamer.client.client.torrents_info.return_value = self._infos() + paths = {"old-v1": self.TARGET, "new-v2": self.V2} + new_queries = 0 + + async def files(torrent_hash): + nonlocal new_queries + if torrent_hash == "new-v2": + new_queries += 1 + if new_queries > 1: + return [] + return [{"name": paths[torrent_hash]}] + + async def rename(torrent_hash, old_path, new_path, verify=True): + assert paths[torrent_hash] == old_path + paths[torrent_hash] = new_path + return RenameResult(RenameOutcome.RENAMED) + + renamer.client.client.torrents_files.side_effect = files + renamer.client.client.torrents_rename_file.side_effect = rename + test_settings.bangumi_manage.rename_method = "pn" + test_settings.bangumi_manage.revision_conflict_policy = "replace" + + with ( + patch("module.manager.renamer.settings", test_settings), + patch.object( + renamer, + "_batch_lookup_offsets", + AsyncMock(return_value=self._offsets()), + ), + ): + assert await renamer.rename() == [] + + assert paths["old-v1"] == self.TARGET + assert len(renamer.events) == 1 + assert "V1 was restored" in renamer.events[0].reason + renamer.client.client.torrents_delete.assert_not_awaited() + + async def test_multifile_owner_blocks_destructive_replacement( + self, renamer, test_settings + ): + renamer.client.client.torrents_info.return_value = self._infos() + + async def files(torrent_hash): + if torrent_hash == "old-v1": + return [{"name": self.TARGET}, {"name": "old-release.nfo"}] + return [{"name": self.V2}] + + renamer.client.client.torrents_files.side_effect = files + test_settings.bangumi_manage.rename_method = "pn" + test_settings.bangumi_manage.revision_conflict_policy = "replace" + + with ( + patch("module.manager.renamer.settings", test_settings), + patch.object( + renamer, + "_batch_lookup_offsets", + AsyncMock(return_value=self._offsets()), + ), + ): + assert await renamer.rename() == [] + + assert len(renamer.events) == 1 + assert "single-file" in renamer.events[0].reason + renamer.client.client.torrents_rename_file.assert_not_awaited() + renamer.client.client.torrents_delete.assert_not_awaited() + + async def test_multi_episode_collection_conflicts_are_persistent( + self, renamer, test_settings + ): + info = { + "hash": "season-pack", + "name": "[ANi] 尼古喵喵 01-02 [1080P].torrent", + "save_path": self.SAVE_PATH, + "tags": "ab:42", + } + renamer.client.client.torrents_info.return_value = [info] + renamer.client.client.torrents_files.return_value = [ + {"name": "raw01.mp4"}, + {"name": "raw02.mp4"}, + ] + renamer.client.client.torrents_rename_file.return_value = RenameResult( + RenameOutcome.DESTINATION_EXISTS, detail="target exists" + ) + test_settings.bangumi_manage.rename_method = "pn" + test_settings.bangumi_manage.remove_bad_torrent = True + test_settings.bangumi_manage.revision_conflict_policy = "replace" + + def parse(torrent_path, **kwargs): + episode = 1 if "01" in torrent_path else 2 + return EpisodeFile( + media_path=torrent_path, + title="尼古喵喵", + season=1, + episode=episode, + suffix=".mp4", + ) + + with ( + patch("module.manager.renamer.settings", test_settings), + patch.object(renamer._parser, "torrent_parser", side_effect=parse), + patch.object( + renamer, + "_batch_lookup_offsets", + AsyncMock(return_value={"season-pack": (0, 0, "episode")}), + ), + ): + assert await renamer.rename() == [] + + assert renamer.client.client.torrents_rename_file.await_count == 2 + assert len(renamer.events) == 2 + renamer.client.client.torrents_delete.assert_not_awaited() + renamer.client.client.set_category.assert_not_awaited() + + restarted = Renamer(renamer.client) + with ( + patch("module.manager.renamer.settings", test_settings), + patch.object(restarted._parser, "torrent_parser", side_effect=parse), + patch.object( + restarted, + "_batch_lookup_offsets", + AsyncMock(return_value={"season-pack": (0, 0, "episode")}), + ), + ): + assert await restarted.rename() == [] + + assert renamer.client.client.torrents_rename_file.await_count == 2 + assert restarted.events == [] + + async def test_restart_finishes_forward_cleanup_after_promotion( + self, renamer, test_settings + ): + infos = self._infos() + renamer.client.client.torrents_info.return_value = infos + paths = {"old-v1": self.TARGET, "new-v2": self.V2} + + async def files(torrent_hash): + return [{"name": paths[torrent_hash]}] + + async def rename(torrent_hash, old_path, new_path, verify=True): + assert paths[torrent_hash] == old_path + paths[torrent_hash] = new_path + return RenameResult(RenameOutcome.RENAMED) + + renamer.client.client.torrents_files.side_effect = files + renamer.client.client.torrents_rename_file.side_effect = rename + renamer.client.client.torrents_delete.return_value = False + test_settings.bangumi_manage.rename_method = "pn" + test_settings.bangumi_manage.revision_conflict_policy = "replace" + + with ( + patch("module.manager.renamer.settings", test_settings), + patch.object( + renamer, + "_batch_lookup_offsets", + AsyncMock(return_value=self._offsets()), + ), + ): + assert await renamer.rename() == [] + + assert paths["new-v2"] == self.TARGET + assert ".ab-replaced-v1-old-v1" in paths["old-v1"] + + from datetime import datetime, timedelta, timezone + + from module.database import Database + + async with Database() as db: + operation = await db.rename_operation.get_by_target( + downloader_type=renamer._downloader_type(), + save_path=self.SAVE_PATH, + target_path=self.TARGET, + ) + assert operation is not None + assert operation.state == "new_promoted" + await db.rename_operation.set_state( + operation.id, + "new_promoted", + retry_at=datetime.now(timezone.utc) - timedelta(seconds=1), + ) + + async def delete(torrent_hash, delete_files=True): + infos[:] = [info for info in infos if info["hash"] != torrent_hash] + return True + + renamer.client.client.torrents_delete.side_effect = delete + restarted = Renamer(renamer.client) + with ( + patch("module.manager.renamer.settings", test_settings), + patch.object( + restarted, + "_batch_lookup_offsets", + AsyncMock(return_value=self._offsets()), + ), + ): + result = await restarted.rename() + + assert len(result) == 1 + assert paths["new-v2"] == self.TARGET + assert renamer.client.client.torrents_delete.await_count == 2 + + # --------------------------------------------------------------------------- # _parse_bangumi_id_from_tags # --------------------------------------------------------------------------- diff --git a/backend/src/test/test_revision_policy.py b/backend/src/test/test_revision_policy.py new file mode 100644 index 00000000..5a37dae2 --- /dev/null +++ b/backend/src/test/test_revision_policy.py @@ -0,0 +1,58 @@ +from module.manager.revision_policy import ( + is_strict_upgrade, + parse_revision_identity, + replacement_staged_path, + same_release_identity, +) + +V1 = "[ANi] 尼古喵喵 - 01 [1080P][Baha][WEB-DL][AAC AVC][CHT].mp4" +V2 = "[ANi] 尼古喵喵 - 01 [V2][1080P][Baha][WEB-DL][AAC AVC][CHT].mp4" + + +def test_mikan_classic_v2_is_a_strict_upgrade(): + old = parse_revision_identity(V1, bangumi_id=42, default_season=1) + new = parse_revision_identity(V2, bangumi_id=42, default_season=1) + + assert old is not None + assert new is not None + assert old.revision == 1 + assert new.revision == 2 + assert same_release_identity(old, new) + assert is_strict_upgrade(old, new) + + +def test_cross_group_or_resolution_is_not_a_strict_upgrade(): + old = parse_revision_identity(V1, bangumi_id=42, default_season=1) + other_group = parse_revision_identity( + V2.replace("[ANi]", "[Other]"), bangumi_id=42, default_season=1 + ) + other_resolution = parse_revision_identity( + V2.replace("1080P", "720P"), bangumi_id=42, default_season=1 + ) + + assert old is not None + assert other_group is not None + assert other_resolution is not None + assert not is_strict_upgrade(old, other_group) + assert not is_strict_upgrade(old, other_resolution) + + +def test_missing_bangumi_group_or_resolution_is_ineligible(): + assert parse_revision_identity(V2, bangumi_id=None, default_season=1) is None + assert ( + parse_revision_identity( + "尼古喵喵 - 01 [V2][1080P].mp4", + bangumi_id=42, + default_season=1, + ) + is None + ) + + +def test_staged_path_is_deterministic_and_keeps_extension(): + assert ( + replacement_staged_path( + "subdir/尼古喵喵 S01E01.mp4", old_task_id="abcdef123456", old_revision=1 + ) + == "subdir/尼古喵喵 S01E01.ab-replaced-v1-abcdef12.mp4" + ) diff --git a/docs/plans/2026-07-13-issue-1078-revision-replacement-design.md b/docs/plans/2026-07-13-issue-1078-revision-replacement-design.md index 14911676..e8e8256a 100644 --- a/docs/plans/2026-07-13-issue-1078-revision-replacement-design.md +++ b/docs/plans/2026-07-13-issue-1078-revision-replacement-design.md @@ -51,6 +51,11 @@ source path, and target path. It stores operation kind, phase, old owner task, deterministic temporary path, parsed identity metadata, attempt counters, retry time, notification time, last error, and timestamps. +Each replacement phase is fenced by an expiring database lease and committed +with compare-and-swap semantics. This supplements the target uniqueness index: +multiple processes may observe the same row, but only the lease holder can run +the phase and persist its transition. + Normal conflict phases are `CONFLICT` and `RETRY_WAIT`. Replacement phases are: ```text @@ -96,11 +101,14 @@ verification after every action. `deleteFiles=true` is allowed only after the single-file guard has passed. For aria2, rename remains an AutoBangumi filesystem move recorded in -`aria2_gid.renamed_paths`. Old-version cleanup must resolve the old gid's current -renamed path, delete that deterministic temporary file, call `forceRemove` -without generic file deletion, and delete the sidecar only after success or an -explicit not-found response. This prevents cleanup from deleting the V2 file now -occupying aria2's original path. +`aria2_gid.renamed_paths`. Before moving a file, aria2 persists an intent with +the source and target paths plus the source filesystem identity. A restart may +finalize the move only when the target still has that exact identity; an +unrelated existing target remains a conflict. Old-version cleanup must resolve +the old gid's current renamed path, delete that deterministic temporary file, +call `forceRemove` without generic file deletion, and delete the sidecar only +after success or an explicit not-found response. This prevents cleanup from +deleting the V2 file now occupying aria2's original path. ## Configuration and UI diff --git a/webui/src/components/layout/ab-notification-center.vue b/webui/src/components/layout/ab-notification-center.vue index bff45af6..3cef074e 100644 --- a/webui/src/components/layout/ab-notification-center.vue +++ b/webui/src/components/layout/ab-notification-center.vue @@ -17,6 +17,7 @@ const KIND_ROUTES: Record = { rss_failure: '/rss', offset_review: '/bangumi', download_failure: '/bangumi', + rename_conflict: '/downloader', }; const SEVERITY_ICONS = { diff --git a/webui/src/components/setting/__tests__/config-manage.test.ts b/webui/src/components/setting/__tests__/config-manage.test.ts new file mode 100644 index 00000000..3ee03a55 --- /dev/null +++ b/webui/src/components/setting/__tests__/config-manage.test.ts @@ -0,0 +1,83 @@ +import { mount } from '@vue/test-utils'; +import { defineComponent, nextTick } from 'vue'; +import ConfigManage from '../config-manage.vue'; + +vi.mock('@/hooks/useMyI18n', () => ({ + useMyI18n: () => ({ t: (key: string) => key }), +})); + +vi.mock('@/store/config', async () => { + const { computed } = await vi.importActual('vue'); + const manageState = { + enable: true, + eps_complete: false, + rename_method: 'pn', + revision_conflict_policy: 'hold', + group_tag: false, + remove_bad_torrent: false, + track_orphans: true, + }; + return { + __manageState: manageState, + useConfigStore: () => ({ + getSettingGroup: () => computed(() => manageState), + }), + }; +}); + +const AbSettingStub = defineComponent({ + name: 'AbSettingStub', + props: { + data: { type: [String, Boolean], default: undefined }, + description: { type: String, default: '' }, + label: { type: [String, Function], required: true }, + prop: { type: Object, default: undefined }, + type: { type: String, required: true }, + }, + emits: ['update:data'], + template: '
', +}); + +describe('config-manage', () => { + it('offers a safe hold default and an explicit higher-revision replacement', async () => { + const wrapper = mount(ConfigManage, { + global: { + stubs: { + 'ab-fold-panel': { template: '
' }, + 'ab-setting': AbSettingStub, + }, + }, + }); + const settings = wrapper.findAllComponents(AbSettingStub); + const policy = settings.find((setting) => { + const label = setting.props('label') as () => string; + return label() === 'config.manage_set.revision_conflict_policy'; + }); + + expect(policy).toBeDefined(); + if (!policy) throw new Error('revision conflict policy setting not found'); + expect(policy.props('data')).toBe('hold'); + expect(policy.props('description')).toBe( + 'config.manage_set.revision_conflict_hint' + ); + expect(policy.props('prop')?.items).toEqual([ + { + id: 1, + label: 'config.manage_set.revision_conflict_hold', + value: 'hold', + }, + { + id: 2, + label: 'config.manage_set.revision_conflict_replace', + value: 'replace', + }, + ]); + + await policy.vm.$emit('update:data', 'replace'); + await nextTick(); + const store = (await import('@/store/config')) as unknown as { + __manageState: { revision_conflict_policy: string }; + }; + expect(store.__manageState.revision_conflict_policy).toBe('replace'); + }); +}); diff --git a/webui/src/components/setting/config-manage.vue b/webui/src/components/setting/config-manage.vue index fbb4daea..6bba5798 100644 --- a/webui/src/components/setting/config-manage.vue +++ b/webui/src/components/setting/config-manage.vue @@ -1,14 +1,32 @@