diff --git a/backend/src/module/downloader/__init__.py b/backend/src/module/downloader/__init__.py index 0aa01bbb..6891579c 100644 --- a/backend/src/module/downloader/__init__.py +++ b/backend/src/module/downloader/__init__.py @@ -3,9 +3,10 @@ from .base import ( DownloaderCapabilities, DownloaderClient, ) -from .download_client import DownloadClient, shutdown +from .download_client import AddResult, DownloadClient, shutdown __all__ = [ + "AddResult", "CoreDownloaderClient", "DownloadClient", "DownloaderCapabilities", diff --git a/backend/src/module/downloader/download_client.py b/backend/src/module/downloader/download_client.py index 032f291a..9fad357b 100644 --- a/backend/src/module/downloader/download_client.py +++ b/backend/src/module/downloader/download_client.py @@ -1,5 +1,6 @@ import asyncio import logging +from enum import Enum from module.conf import settings from module.models import Bangumi, Torrent @@ -11,6 +12,20 @@ from .path import gen_save_path logger = logging.getLogger(__name__) +class AddResult(Enum): + """``add_torrent`` 的三态结果。 + + - ``ADDED``: 真正新增了下载任务； + - ``DUPLICATE``: 下载器里已有同一种子（qB 重复 / aria2 去重命中）， + 对调用方来说等同于成功，不应触发失败通知； + - ``FAILED``: 投递失败（抓取种子文件失败或客户端异常），可以重试。 + """ + + ADDED = "added" + DUPLICATE = "duplicate" + FAILED = "failed" + + # --------------------------------------------------------------------------- # Module-level concrete-client cache (session reuse, #1039 / #900) # @@ -271,12 +286,15 @@ class DownloadClient: return await self.client.torrents_resume(hashes) - async def add_torrent(self, torrent: Torrent | list, bangumi: Bangumi) -> bool: + async def add_torrent(self, torrent: Torrent | list, bangumi: Bangumi) -> AddResult: """Download a torrent (or list of torrents) for the given bangumi entry. Handles both magnet links and .torrent file URLs, fetching file bytes when necessary. Tags each torrent with ``ab:`` for later episode-offset lookup during rename. + + 返回 :class:`AddResult`，区分"新增成功 / 已添加过 / 投递失败"， + 调用方据此决定是否重试或发送失败通知。 """ if not bangumi.save_path: bangumi.save_path = gen_save_path(bangumi) @@ -287,7 +305,7 @@ class DownloadClient: logger.debug( "[Downloader] No torrent found: %s", bangumi.official_title ) - return False + return AddResult.FAILED if "magnet" in torrent[0].url: torrent_url = [t.url for t in torrent] torrent_file = None @@ -301,7 +319,7 @@ class DownloadClient: logger.warning( f"[Downloader] Failed to fetch torrent files for: {bangumi.official_title}" ) - return False + return AddResult.FAILED torrent_url = None else: if "magnet" in torrent.url: @@ -313,7 +331,7 @@ class DownloadClient: logger.warning( f"[Downloader] Failed to fetch torrent file for: {bangumi.official_title}" ) - return False + return AddResult.FAILED torrent_url = None # Create tag with bangumi_id for offset lookup during rename tags = f"ab:{bangumi.id}" if bangumi.id else None @@ -326,17 +344,17 @@ class DownloadClient: tags=tags, ): logger.debug("[Downloader] Add torrent: %s", bangumi.official_title) - return True + return AddResult.ADDED else: logger.debug( "[Downloader] Torrent added before: %s", bangumi.official_title ) - return False + return AddResult.DUPLICATE except Exception as e: logger.error( f"[Downloader] Failed to add torrent for {bangumi.official_title}: {e}" ) - return False + return AddResult.FAILED async def move_torrent(self, hashes, location): if not self._supports("can_manage", "move_torrent"): diff --git a/backend/src/module/manager/collector.py b/backend/src/module/manager/collector.py index d0fdacfe..2238f4e2 100644 --- a/backend/src/module/manager/collector.py +++ b/backend/src/module/manager/collector.py @@ -1,7 +1,7 @@ import logging from module.database import Database -from module.downloader import DownloadClient +from module.downloader import AddResult, DownloadClient from module.models import Bangumi, ResponseModel from module.network import RequestContent from module.rss import RSSEngine @@ -27,7 +27,7 @@ class SeasonCollector: link, bangumi.filter.replace(",", "|") ) async with Database() as db: - if await self.client.add_torrent(torrents, bangumi): + if await self.client.add_torrent(torrents, bangumi) is AddResult.ADDED: logger.info( f"Collections of {bangumi.official_title} Season {bangumi.season} completed." ) diff --git a/backend/src/module/rss/engine.py b/backend/src/module/rss/engine.py index ff91e0e2..91e088b7 100644 --- a/backend/src/module/rss/engine.py +++ b/backend/src/module/rss/engine.py @@ -8,7 +8,7 @@ from urllib.parse import urlparse from module.database import Database from module.database.bangumi import _groups_are_similar, match_bangumi_in_list -from module.downloader import DownloadClient +from module.downloader import AddResult, DownloadClient from module.models import Bangumi, Episode, ResponseModel, RSSItem, Torrent from module.network import RequestContent from module.notification.events import ( @@ -168,11 +168,15 @@ class RSSEngine: ) -> Optional[Bangumi]: matched = match_bangumi_in_list(torrent.name, bangumi_list) if matched: - torrent.bangumi_id = matched.id + # 只有通过排除过滤的种子才关联 bangumi_id：被过滤掉的种子 + # 不能挂在番剧名下，否则 OffsetScanner 会用用户明确排除的 + # 剧集来计算 offset 建议。 if matched.filter == "": + torrent.bangumi_id = matched.id return matched pattern = self._get_filter_pattern(matched.filter) if not pattern.search(torrent.name): + torrent.bangumi_id = matched.id return matched return None @@ -336,6 +340,7 @@ class RSSEngine: error=error, ) ) + failed_ids: set[int] = set() for torrent, matched_data in zip(new_torrents, matches): if matched_data: if id(torrent) in skip_ids: @@ -345,22 +350,32 @@ class RSSEngine: torrent.name, matched_data.official_title, ) - elif await client.add_torrent(torrent, matched_data): - logger.debug("[Engine] Add torrent %s to client", torrent.name) - torrent.downloaded = True - else: - # add_torrent() already retried the underlying fetch - # internally (RequestURL.get_url, retry=3); leave - # downloaded=False so a future tick can retry, and - # surface the failure now. + continue + result = await client.add_torrent(torrent, matched_data) + if result is AddResult.FAILED: + # 投递失败：不入库（check_new 按 URL 去重，入库就 + # 永远不会再处理），下一轮 refresh 该种子仍在源里 + # 时会重新匹配并重试；同时发出失败通知。 + failed_ids.add(id(torrent)) events.append( DownloadFailureEvent( official_title=matched_data.official_title, torrent_name=torrent.name, ) ) - # Add all torrents to database - await self.db.torrent.add_all(new_torrents) + else: + # ADDED 与 DUPLICATE（下载器里已有同一种子）都视为 + # 成功，不再对健康的重复种子发失败通知。 + logger.debug( + "[Engine] Add torrent %s to client (%s)", + torrent.name, + result.value, + ) + torrent.downloaded = True + # Add all torrents to database (投递失败的除外，留待重试) + await self.db.torrent.add_all( + [t for t in new_torrents if id(t) not in failed_ids] + ) await self.db.commit() return events diff --git a/backend/src/test/test_download_client.py b/backend/src/test/test_download_client.py index 76271ac9..2e2d6484 100644 --- a/backend/src/test/test_download_client.py +++ b/backend/src/test/test_download_client.py @@ -4,7 +4,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest -from module.downloader.download_client import DownloadClient +from module.downloader.download_client import AddResult, DownloadClient from module.models import Bangumi, Torrent from module.models.config import Config from test.factories import make_bangumi, make_torrent @@ -80,7 +80,7 @@ class TestAddTorrent: result = await download_client.add_torrent(torrent, bangumi) - assert result is True + assert result is AddResult.ADDED call_kwargs = mock_qb_client.add_torrents.call_args[1] assert call_kwargs["torrent_urls"] == "magnet:?xt=urn:btih:abc123" assert call_kwargs["torrent_files"] is None @@ -98,7 +98,7 @@ class TestAddTorrent: result = await download_client.add_torrent(torrent, bangumi) - assert result is True + assert result is AddResult.ADDED call_kwargs = mock_qb_client.add_torrents.call_args[1] assert call_kwargs["torrent_files"] == b"torrent-file-data" assert call_kwargs["torrent_urls"] is None @@ -119,12 +119,14 @@ class TestAddTorrent: result = await download_client.add_torrent(torrents, bangumi) - assert result is True + assert result is AddResult.ADDED call_kwargs = mock_qb_client.add_torrents.call_args[1] assert len(call_kwargs["torrent_urls"]) == 3 - async def test_empty_list_returns_false(self, download_client, mock_qb_client): - """Empty torrent list returns False without calling client.""" + async def test_add_torrent_empty_list_returns_failed( + self, download_client, mock_qb_client + ): + """Empty torrent list returns FAILED without calling client.""" bangumi = make_bangumi() with patch("module.downloader.download_client.RequestContent") as MockReq: mock_req = AsyncMock() @@ -132,11 +134,14 @@ class TestAddTorrent: MockReq.return_value.__aexit__ = AsyncMock(return_value=False) result = await download_client.add_torrent([], bangumi) - assert result is False + assert result is AddResult.FAILED mock_qb_client.add_torrents.assert_not_called() - async def test_client_rejects_returns_false(self, download_client, mock_qb_client): - """When client.add_torrents returns False, returns False.""" + async def test_add_torrent_client_reports_existing_returns_duplicate( + self, download_client, mock_qb_client + ): + """When client.add_torrents returns False (already added), returns + DUPLICATE, not a failure.""" mock_qb_client.add_torrents.return_value = False torrent = make_torrent(url="magnet:?xt=urn:btih:abc") bangumi = make_bangumi() @@ -148,9 +153,46 @@ class TestAddTorrent: result = await download_client.add_torrent(torrent, bangumi) - assert result is False + assert result is AddResult.DUPLICATE - async def test_generates_save_path_if_missing(self, download_client, mock_qb_client): + async def test_add_torrent_fetch_failure_returns_failed( + self, download_client, mock_qb_client + ): + """When the .torrent file can't be fetched, returns FAILED.""" + torrent = make_torrent(url="https://example.com/file.torrent") + bangumi = make_bangumi() + + with patch("module.downloader.download_client.RequestContent") as MockReq: + mock_req = AsyncMock() + mock_req.get_content = AsyncMock(return_value=None) + MockReq.return_value.__aenter__ = AsyncMock(return_value=mock_req) + MockReq.return_value.__aexit__ = AsyncMock(return_value=False) + + result = await download_client.add_torrent(torrent, bangumi) + + assert result is AddResult.FAILED + mock_qb_client.add_torrents.assert_not_called() + + async def test_add_torrent_client_exception_returns_failed( + self, download_client, mock_qb_client + ): + """When client.add_torrents raises, returns FAILED.""" + mock_qb_client.add_torrents.side_effect = RuntimeError("connection lost") + torrent = make_torrent(url="magnet:?xt=urn:btih:abc") + bangumi = make_bangumi() + + with patch("module.downloader.download_client.RequestContent") as MockReq: + mock_req = AsyncMock() + MockReq.return_value.__aenter__ = AsyncMock(return_value=mock_req) + MockReq.return_value.__aexit__ = AsyncMock(return_value=False) + + result = await download_client.add_torrent(torrent, bangumi) + + assert result is AddResult.FAILED + + async def test_generates_save_path_if_missing( + self, download_client, mock_qb_client + ): """When bangumi.save_path is empty, generates one.""" torrent = make_torrent(url="magnet:?xt=urn:btih:abc") bangumi = make_bangumi(save_path=None) @@ -187,13 +229,17 @@ class TestClientDelegation: async def test_rename_torrent_file_success(self, download_client, mock_qb_client): """rename_torrent_file returns True on success.""" mock_qb_client.torrents_rename_file.return_value = True - result = await download_client.rename_torrent_file("hash1", "old.mkv", "new.mkv") + result = await download_client.rename_torrent_file( + "hash1", "old.mkv", "new.mkv" + ) assert result is True async def test_rename_torrent_file_failure(self, download_client, mock_qb_client): """rename_torrent_file returns False on failure.""" mock_qb_client.torrents_rename_file.return_value = False - result = await download_client.rename_torrent_file("hash1", "old.mkv", "new.mkv") + result = await download_client.rename_torrent_file( + "hash1", "old.mkv", "new.mkv" + ) assert result is False async def test_rename_torrent_file_passes_verify_flag( @@ -210,7 +256,9 @@ class TestClientDelegation: async def test_delete_torrent(self, download_client, mock_qb_client): """delete_torrent delegates to client.torrents_delete.""" await download_client.delete_torrent("hash1", delete_files=True) - mock_qb_client.torrents_delete.assert_called_once_with("hash1", delete_files=True) + mock_qb_client.torrents_delete.assert_called_once_with( + "hash1", delete_files=True + ) # --------------------------------------------------------------------------- @@ -247,7 +295,9 @@ class TestContextManagerAuth: with pytest.raises(ConnectionError, match="authentication failed"): await download_client.__aenter__() - async def test_aenter_succeeds_when_auth_passes(self, download_client, mock_qb_client): + async def test_aenter_succeeds_when_auth_passes( + self, download_client, mock_qb_client + ): """__aenter__ returns self when auth succeeds.""" mock_qb_client.auth.return_value = True download_client.authed = False diff --git a/backend/src/test/test_integration.py b/backend/src/test/test_integration.py index a819becc..ba75ce26 100644 --- a/backend/src/test/test_integration.py +++ b/backend/src/test/test_integration.py @@ -3,6 +3,7 @@ from unittest.mock import AsyncMock, patch from module.database import Database +from module.downloader import AddResult from module.models import EpisodeFile, Torrent from module.rss.engine import RSSEngine from test.factories import make_bangumi, make_rss_item @@ -54,7 +55,7 @@ class TestRssToDownloadFlow: # 4. Mock download client mock_client = AsyncMock() - mock_client.add_torrent = AsyncMock(return_value=True) + mock_client.add_torrent = AsyncMock(return_value=AddResult.ADDED) # 5. Execute refresh_rss await engine.refresh_rss(mock_client) @@ -107,7 +108,7 @@ class TestRssToDownloadFlow: ) as mock_get: mock_get.return_value = torrents mock_client = AsyncMock() - mock_client.add_torrent = AsyncMock(return_value=True) + mock_client.add_torrent = AsyncMock(return_value=AddResult.ADDED) await engine.refresh_rss(mock_client) # Only 1080p should be downloaded (720p is filtered) @@ -147,7 +148,7 @@ class TestRssToDownloadFlow: ) as mock_get: mock_get.return_value = torrents mock_client = AsyncMock() - mock_client.add_torrent = AsyncMock(return_value=True) + mock_client.add_torrent = AsyncMock(return_value=AddResult.ADDED) await engine.refresh_rss(mock_client) # Only ep02 should be downloaded (ep01 already exists) diff --git a/backend/src/test/test_rss_engine_new.py b/backend/src/test/test_rss_engine_new.py index 9e34fecb..d6b4f4a4 100644 --- a/backend/src/test/test_rss_engine_new.py +++ b/backend/src/test/test_rss_engine_new.py @@ -7,6 +7,7 @@ import pytest import pytest_asyncio from module.database import Database +from module.downloader import AddResult from module.models import Torrent from module.notification.events import DownloadFailureEvent, RssFailureEvent from module.rss.engine import RSSEngine @@ -185,6 +186,48 @@ class TestMatchTorrent: assert result2 is not None + async def test_match_torrent_filter_rejected_leaves_bangumi_id_unset( + self, rss_engine + ): + """A filter-rejected torrent must NOT be associated with the bangumi, + or OffsetScanner would compute offsets from excluded episodes.""" + bangumi = make_bangumi(title_raw="Mushoku Tensei", filter="720") + await rss_engine.db.bangumi.add(bangumi) + + torrent = make_torrent(name="[Sub] Mushoku Tensei - 01 [720p].mkv") + result = rss_engine.match_torrent( + torrent, await rss_engine.db.bangumi.search_all() + ) + + assert result is None + assert torrent.bangumi_id is None + + async def test_match_torrent_filter_passed_sets_bangumi_id(self, rss_engine): + """A torrent passing a non-empty filter gets bangumi_id set.""" + bangumi = make_bangumi(title_raw="Mushoku Tensei", filter="720") + await rss_engine.db.bangumi.add(bangumi) + + torrent = make_torrent(name="[Sub] Mushoku Tensei - 01 [1080p].mkv") + result = rss_engine.match_torrent( + torrent, await rss_engine.db.bangumi.search_all() + ) + + assert result is not None + assert torrent.bangumi_id == result.id + + async def test_match_torrent_empty_filter_sets_bangumi_id(self, rss_engine): + """A torrent matched with an empty filter also gets bangumi_id set.""" + bangumi = make_bangumi(title_raw="Mushoku Tensei", filter="") + await rss_engine.db.bangumi.add(bangumi) + + torrent = make_torrent(name="[Sub] Mushoku Tensei - 01 [1080p].mkv") + result = rss_engine.match_torrent( + torrent, await rss_engine.db.bangumi.search_all() + ) + + assert result is not None + assert torrent.bangumi_id == result.id + # --------------------------------------------------------------------------- # refresh_rss @@ -212,7 +255,7 @@ class TestRefreshRss: # Create a mock client client = AsyncMock() - client.add_torrent = AsyncMock(return_value=True) + client.add_torrent = AsyncMock(return_value=AddResult.ADDED) await rss_engine.refresh_rss(client) @@ -346,7 +389,7 @@ class TestRefreshRssEvents: ) as mock_get: mock_get.return_value = [new_torrent] client = AsyncMock() - client.add_torrent = AsyncMock(return_value=False) + client.add_torrent = AsyncMock(return_value=AddResult.FAILED) events = await rss_engine.refresh_rss(client) @@ -355,8 +398,9 @@ class TestRefreshRssEvents: assert events[0].official_title == bangumi.official_title assert events[0].torrent_name == new_torrent.name + # The failed torrent is NOT persisted, so a later tick can retry it. all_torrents = await rss_engine.db.torrent.search_all() - assert all_torrents[0].downloaded is False + assert all_torrents == [] async def test_add_torrent_success_returns_no_download_failure_event( self, rss_engine, mock_qb_client @@ -376,12 +420,116 @@ class TestRefreshRssEvents: ) as mock_get: mock_get.return_value = [new_torrent] client = AsyncMock() - client.add_torrent = AsyncMock(return_value=True) + client.add_torrent = AsyncMock(return_value=AddResult.ADDED) events = await rss_engine.refresh_rss(client) assert events == [] + async def test_add_torrent_duplicate_marks_downloaded_without_event( + self, rss_engine + ): + """A torrent the client already has (DUPLICATE) is treated as success: + no failure notification, row persisted as downloaded.""" + rss_item = make_rss_item(enabled=True) + await rss_engine.db.rss.add(rss_item) + bangumi = make_bangumi(title_raw="Mushoku Tensei", filter="") + await rss_engine.db.bangumi.add(bangumi) + + new_torrent = Torrent( + name="[Sub] Mushoku Tensei - 12 [1080p].mkv", + url="https://example.com/ep12.torrent", + ) + with patch.object( + RSSEngine, "_get_torrents", new_callable=AsyncMock + ) as mock_get: + mock_get.return_value = [new_torrent] + client = AsyncMock() + client.add_torrent = AsyncMock(return_value=AddResult.DUPLICATE) + + events = await rss_engine.refresh_rss(client) + + assert events == [] + all_torrents = await rss_engine.db.torrent.search_all() + assert len(all_torrents) == 1 + assert all_torrents[0].downloaded is True + + +# --------------------------------------------------------------------------- +# refresh_rss retry semantics for failed adds +# --------------------------------------------------------------------------- + + +class TestRefreshRssRetry: + async def test_refresh_rss_failed_add_retried_on_next_tick(self, rss_engine): + """A torrent whose add FAILED is re-attempted on the next refresh + (it must not be persisted, or check_new would filter it forever).""" + rss_item = make_rss_item(enabled=True) + await rss_engine.db.rss.add(rss_item) + bangumi = make_bangumi(title_raw="Mushoku Tensei", filter="") + await rss_engine.db.bangumi.add(bangumi) + + def feed_torrent(): + return Torrent( + name="[Sub] Mushoku Tensei - 12 [1080p].mkv", + url="https://example.com/ep12.torrent", + ) + + with patch.object( + RSSEngine, "_get_torrents", new_callable=AsyncMock + ) as mock_get: + client = AsyncMock() + + # Tick 1: transient failure + mock_get.return_value = [feed_torrent()] + client.add_torrent = AsyncMock(return_value=AddResult.FAILED) + await rss_engine.refresh_rss(client) + client.add_torrent.assert_called_once() + assert await rss_engine.db.torrent.search_all() == [] + + # Tick 2: same feed item is still there, add now succeeds + mock_get.return_value = [feed_torrent()] + client.add_torrent = AsyncMock(return_value=AddResult.ADDED) + await rss_engine.refresh_rss(client) + client.add_torrent.assert_called_once() + + all_torrents = await rss_engine.db.torrent.search_all() + assert len(all_torrents) == 1 + assert all_torrents[0].downloaded is True + + async def test_refresh_rss_filter_rejected_persisted_and_not_retried( + self, rss_engine + ): + """Filter-rejected torrents keep the old behavior: persisted once + (downloaded=False, no bangumi association) and never re-processed.""" + rss_item = make_rss_item(enabled=True) + await rss_engine.db.rss.add(rss_item) + bangumi = make_bangumi(title_raw="Mushoku Tensei", filter="720") + await rss_engine.db.bangumi.add(bangumi) + + def feed_torrent(): + return Torrent( + name="[Sub] Mushoku Tensei - 01 [720p].mkv", + url="https://example.com/ep01-720.torrent", + ) + + with patch.object( + RSSEngine, "_get_torrents", new_callable=AsyncMock + ) as mock_get: + client = AsyncMock() + client.add_torrent = AsyncMock(return_value=AddResult.ADDED) + + mock_get.return_value = [feed_torrent()] + await rss_engine.refresh_rss(client) + mock_get.return_value = [feed_torrent()] + await rss_engine.refresh_rss(client) + + client.add_torrent.assert_not_called() + all_torrents = await rss_engine.db.torrent.search_all() + assert len(all_torrents) == 1 + assert all_torrents[0].downloaded is False + assert all_torrents[0].bangumi_id is None + # --------------------------------------------------------------------------- # add_rss @@ -714,7 +862,7 @@ class TestRefreshRssPreferenceDedup: ) as mock_get: mock_get.return_value = [preferred, other] client = AsyncMock() - client.add_torrent = AsyncMock(return_value=True) + client.add_torrent = AsyncMock(return_value=AddResult.ADDED) await rss_engine.refresh_rss(client) @@ -749,7 +897,7 @@ class TestRefreshRssPreferenceDedup: ) as mock_get: mock_get.return_value = [torrent_a, torrent_b] client = AsyncMock() - client.add_torrent = AsyncMock(return_value=True) + client.add_torrent = AsyncMock(return_value=AddResult.ADDED) await rss_engine.refresh_rss(client)