nas/Auto_Bangumi
1
0
Fork 0
mirror of https://github.com/EstrellaXD/Auto_Bangumi.git synced 2026-10-31 12:45:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(rss): retry failed adds, stop spurious failure alerts, unpollute offset scan

Browse Source 
- Failed torrent adds are no longer persisted, so the feed item
  re-enters the pipeline on the next tick — the previous
  downloaded=False row blocked retry forever via check_new's URL dedup
- add_torrent returns AddResult (ADDED/DUPLICATE/FAILED); healthy
  duplicates no longer emit download-failure notifications
- bangumi_id is set only on torrents that pass the exclusion filter,
  so OffsetScanner no longer computes suggestions from excluded
  episodes

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GTnuEgR78cBk32bLNKar3z
This commit is contained in:
Estrella Pan
2026-07-02 22:07:12 +02:00
parent 8597e3d64f
commit eada4e7210
7 changed files with 279 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
backend/src/module/downloader/__init__.py
View File

@@ -3,9 +3,10 @@ from .base import (
DownloaderCapabilities,
DownloaderClient,
)
from .download_client import DownloadClient, shutdown
from .download_client import AddResult, DownloadClient, shutdown
__all__ = [
"AddResult",
"CoreDownloaderClient",
"DownloadClient",
"DownloaderCapabilities",

32
backend/src/module/downloader/download_client.py
View File

@@ -1,5 +1,6 @@
import asyncio
import logging
from enum import Enum
from module.conf import settings
from module.models import Bangumi, Torrent
@@ -11,6 +12,20 @@ from .path import gen_save_path
logger = logging.getLogger(__name__)
class AddResult(Enum):
"""``add_torrent`` 的三态结果。
- ``ADDED``: 真正新增了下载任务；
- ``DUPLICATE``: 下载器里已有同一种子（qB 重复 / aria2 去重命中），
对调用方来说等同于成功，不应触发失败通知；
- ``FAILED``: 投递失败（抓取种子文件失败或客户端异常），可以重试。
"""
ADDED = "added"
DUPLICATE = "duplicate"
FAILED = "failed"
# ---------------------------------------------------------------------------
# Module-level concrete-client cache (session reuse, #1039 / #900)
#
@@ -271,12 +286,15 @@ class DownloadClient:
return
await self.client.torrents_resume(hashes)
async def add_torrent(self, torrent: Torrent | list, bangumi: Bangumi) -> bool:
async def add_torrent(self, torrent: Torrent | list, bangumi: Bangumi) -> AddResult:
"""Download a torrent (or list of torrents) for the given bangumi entry.
Handles both magnet links and .torrent file URLs, fetching file bytes
when necessary. Tags each torrent with ``ab:<bangumi_id>`` for later
episode-offset lookup during rename.
返回 :class:`AddResult`，区分"新增成功 / 已添加过 / 投递失败"，
调用方据此决定是否重试或发送失败通知。
"""
if not bangumi.save_path:
bangumi.save_path = gen_save_path(bangumi)
@@ -287,7 +305,7 @@ class DownloadClient:
logger.debug(
"[Downloader] No torrent found: %s", bangumi.official_title
)
return False
return AddResult.FAILED
if "magnet" in torrent[0].url:
torrent_url = [t.url for t in torrent]
torrent_file = None
@@ -301,7 +319,7 @@ class DownloadClient:
logger.warning(
f"[Downloader] Failed to fetch torrent files for: {bangumi.official_title}"
)
return False
return AddResult.FAILED
torrent_url = None
else:
if "magnet" in torrent.url:
@@ -313,7 +331,7 @@ class DownloadClient:
logger.warning(
f"[Downloader] Failed to fetch torrent file for: {bangumi.official_title}"
)
return False
return AddResult.FAILED
torrent_url = None
# Create tag with bangumi_id for offset lookup during rename
tags = f"ab:{bangumi.id}" if bangumi.id else None
@@ -326,17 +344,17 @@ class DownloadClient:
tags=tags,
):
logger.debug("[Downloader] Add torrent: %s", bangumi.official_title)
return True
return AddResult.ADDED
else:
logger.debug(
"[Downloader] Torrent added before: %s", bangumi.official_title
)
return False
return AddResult.DUPLICATE
except Exception as e:
logger.error(
f"[Downloader] Failed to add torrent for {bangumi.official_title}: {e}"
)
return False
return AddResult.FAILED
async def move_torrent(self, hashes, location):
if not self._supports("can_manage", "move_torrent"):

4
backend/src/module/manager/collector.py
View File

@@ -1,7 +1,7 @@
import logging
from module.database import Database
from module.downloader import DownloadClient
from module.downloader import AddResult, DownloadClient
from module.models import Bangumi, ResponseModel
from module.network import RequestContent
from module.rss import RSSEngine
@@ -27,7 +27,7 @@ class SeasonCollector:
link, bangumi.filter.replace(",", "|")
)
async with Database() as db:
if await self.client.add_torrent(torrents, bangumi):
if await self.client.add_torrent(torrents, bangumi) is AddResult.ADDED:
logger.info(
f"Collections of {bangumi.official_title} Season {bangumi.season} completed."
)

39
backend/src/module/rss/engine.py
View File

@@ -8,7 +8,7 @@ from urllib.parse import urlparse
from module.database import Database
from module.database.bangumi import _groups_are_similar, match_bangumi_in_list
from module.downloader import DownloadClient
from module.downloader import AddResult, DownloadClient
from module.models import Bangumi, Episode, ResponseModel, RSSItem, Torrent
from module.network import RequestContent
from module.notification.events import (
@@ -168,11 +168,15 @@ class RSSEngine:
) -> Optional[Bangumi]:
matched = match_bangumi_in_list(torrent.name, bangumi_list)
if matched:
torrent.bangumi_id = matched.id
# 只有通过排除过滤的种子才关联 bangumi_id：被过滤掉的种子
# 不能挂在番剧名下，否则 OffsetScanner 会用用户明确排除的
# 剧集来计算 offset 建议。
if matched.filter == "":
torrent.bangumi_id = matched.id
return matched
pattern = self._get_filter_pattern(matched.filter)
if not pattern.search(torrent.name):
torrent.bangumi_id = matched.id
return matched
return None
@@ -336,6 +340,7 @@ class RSSEngine:
error=error,
)
)
failed_ids: set[int] = set()
for torrent, matched_data in zip(new_torrents, matches):
if matched_data:
if id(torrent) in skip_ids:
@@ -345,22 +350,32 @@ class RSSEngine:
torrent.name,
matched_data.official_title,
)
elif await client.add_torrent(torrent, matched_data):
logger.debug("[Engine] Add torrent %s to client", torrent.name)
torrent.downloaded = True
else:
# add_torrent() already retried the underlying fetch
# internally (RequestURL.get_url, retry=3); leave
# downloaded=False so a future tick can retry, and
# surface the failure now.
continue
result = await client.add_torrent(torrent, matched_data)
if result is AddResult.FAILED:
# 投递失败：不入库（check_new 按 URL 去重，入库就
# 永远不会再处理），下一轮 refresh 该种子仍在源里
# 时会重新匹配并重试；同时发出失败通知。
failed_ids.add(id(torrent))
events.append(
DownloadFailureEvent(
official_title=matched_data.official_title,
torrent_name=torrent.name,
)
)
# Add all torrents to database
await self.db.torrent.add_all(new_torrents)
else:
# ADDED 与 DUPLICATE（下载器里已有同一种子）都视为
# 成功，不再对健康的重复种子发失败通知。
logger.debug(
"[Engine] Add torrent %s to client (%s)",
torrent.name,
result.value,
)
torrent.downloaded = True
# Add all torrents to database (投递失败的除外，留待重试)
await self.db.torrent.add_all(
[t for t in new_torrents if id(t) not in failed_ids]
)
await self.db.commit()
return events

80
backend/src/test/test_download_client.py
View File

@@ -4,7 +4,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from module.downloader.download_client import DownloadClient
from module.downloader.download_client import AddResult, DownloadClient
from module.models import Bangumi, Torrent
from module.models.config import Config
from test.factories import make_bangumi, make_torrent
@@ -80,7 +80,7 @@ class TestAddTorrent:
result = await download_client.add_torrent(torrent, bangumi)
assert result is True
assert result is AddResult.ADDED
call_kwargs = mock_qb_client.add_torrents.call_args[1]
assert call_kwargs["torrent_urls"] == "magnet:?xt=urn:btih:abc123"
assert call_kwargs["torrent_files"] is None
@@ -98,7 +98,7 @@ class TestAddTorrent:
result = await download_client.add_torrent(torrent, bangumi)
assert result is True
assert result is AddResult.ADDED
call_kwargs = mock_qb_client.add_torrents.call_args[1]
assert call_kwargs["torrent_files"] == b"torrent-file-data"
assert call_kwargs["torrent_urls"] is None
@@ -119,12 +119,14 @@ class TestAddTorrent:
result = await download_client.add_torrent(torrents, bangumi)
assert result is True
assert result is AddResult.ADDED
call_kwargs = mock_qb_client.add_torrents.call_args[1]
assert len(call_kwargs["torrent_urls"]) == 3
async def test_empty_list_returns_false(self, download_client, mock_qb_client):
"""Empty torrent list returns False without calling client."""
async def test_add_torrent_empty_list_returns_failed(
self, download_client, mock_qb_client
):
"""Empty torrent list returns FAILED without calling client."""
bangumi = make_bangumi()
with patch("module.downloader.download_client.RequestContent") as MockReq:
mock_req = AsyncMock()
@@ -132,11 +134,14 @@ class TestAddTorrent:
MockReq.return_value.__aexit__ = AsyncMock(return_value=False)
result = await download_client.add_torrent([], bangumi)
assert result is False
assert result is AddResult.FAILED
mock_qb_client.add_torrents.assert_not_called()
async def test_client_rejects_returns_false(self, download_client, mock_qb_client):
"""When client.add_torrents returns False, returns False."""
async def test_add_torrent_client_reports_existing_returns_duplicate(
self, download_client, mock_qb_client
):
"""When client.add_torrents returns False (already added), returns
DUPLICATE, not a failure."""
mock_qb_client.add_torrents.return_value = False
torrent = make_torrent(url="magnet:?xt=urn:btih:abc")
bangumi = make_bangumi()
@@ -148,9 +153,46 @@ class TestAddTorrent:
result = await download_client.add_torrent(torrent, bangumi)
assert result is False
assert result is AddResult.DUPLICATE
async def test_generates_save_path_if_missing(self, download_client, mock_qb_client):
async def test_add_torrent_fetch_failure_returns_failed(
self, download_client, mock_qb_client
):
"""When the .torrent file can't be fetched, returns FAILED."""
torrent = make_torrent(url="https://example.com/file.torrent")
bangumi = make_bangumi()
with patch("module.downloader.download_client.RequestContent") as MockReq:
mock_req = AsyncMock()
mock_req.get_content = AsyncMock(return_value=None)
MockReq.return_value.__aenter__ = AsyncMock(return_value=mock_req)
MockReq.return_value.__aexit__ = AsyncMock(return_value=False)
result = await download_client.add_torrent(torrent, bangumi)
assert result is AddResult.FAILED
mock_qb_client.add_torrents.assert_not_called()
async def test_add_torrent_client_exception_returns_failed(
self, download_client, mock_qb_client
):
"""When client.add_torrents raises, returns FAILED."""
mock_qb_client.add_torrents.side_effect = RuntimeError("connection lost")
torrent = make_torrent(url="magnet:?xt=urn:btih:abc")
bangumi = make_bangumi()
with patch("module.downloader.download_client.RequestContent") as MockReq:
mock_req = AsyncMock()
MockReq.return_value.__aenter__ = AsyncMock(return_value=mock_req)
MockReq.return_value.__aexit__ = AsyncMock(return_value=False)
result = await download_client.add_torrent(torrent, bangumi)
assert result is AddResult.FAILED
async def test_generates_save_path_if_missing(
self, download_client, mock_qb_client
):
"""When bangumi.save_path is empty, generates one."""
torrent = make_torrent(url="magnet:?xt=urn:btih:abc")
bangumi = make_bangumi(save_path=None)
@@ -187,13 +229,17 @@ class TestClientDelegation:
async def test_rename_torrent_file_success(self, download_client, mock_qb_client):
"""rename_torrent_file returns True on success."""
mock_qb_client.torrents_rename_file.return_value = True
result = await download_client.rename_torrent_file("hash1", "old.mkv", "new.mkv")
result = await download_client.rename_torrent_file(
"hash1", "old.mkv", "new.mkv"
)
assert result is True
async def test_rename_torrent_file_failure(self, download_client, mock_qb_client):
"""rename_torrent_file returns False on failure."""
mock_qb_client.torrents_rename_file.return_value = False
result = await download_client.rename_torrent_file("hash1", "old.mkv", "new.mkv")
result = await download_client.rename_torrent_file(
"hash1", "old.mkv", "new.mkv"
)
assert result is False
async def test_rename_torrent_file_passes_verify_flag(
@@ -210,7 +256,9 @@ class TestClientDelegation:
async def test_delete_torrent(self, download_client, mock_qb_client):
"""delete_torrent delegates to client.torrents_delete."""
await download_client.delete_torrent("hash1", delete_files=True)
mock_qb_client.torrents_delete.assert_called_once_with("hash1", delete_files=True)
mock_qb_client.torrents_delete.assert_called_once_with(
"hash1", delete_files=True
)
# ---------------------------------------------------------------------------
@@ -247,7 +295,9 @@ class TestContextManagerAuth:
with pytest.raises(ConnectionError, match="authentication failed"):
await download_client.__aenter__()
async def test_aenter_succeeds_when_auth_passes(self, download_client, mock_qb_client):
async def test_aenter_succeeds_when_auth_passes(
self, download_client, mock_qb_client
):
"""__aenter__ returns self when auth succeeds."""
mock_qb_client.auth.return_value = True
download_client.authed = False

7
backend/src/test/test_integration.py
View File

@@ -3,6 +3,7 @@
from unittest.mock import AsyncMock, patch
from module.database import Database
from module.downloader import AddResult
from module.models import EpisodeFile, Torrent
from module.rss.engine import RSSEngine
from test.factories import make_bangumi, make_rss_item
@@ -54,7 +55,7 @@ class TestRssToDownloadFlow:
# 4. Mock download client
mock_client = AsyncMock()
mock_client.add_torrent = AsyncMock(return_value=True)
mock_client.add_torrent = AsyncMock(return_value=AddResult.ADDED)
# 5. Execute refresh_rss
await engine.refresh_rss(mock_client)
@@ -107,7 +108,7 @@ class TestRssToDownloadFlow:
) as mock_get:
mock_get.return_value = torrents
mock_client = AsyncMock()
mock_client.add_torrent = AsyncMock(return_value=True)
mock_client.add_torrent = AsyncMock(return_value=AddResult.ADDED)
await engine.refresh_rss(mock_client)
# Only 1080p should be downloaded (720p is filtered)
@@ -147,7 +148,7 @@ class TestRssToDownloadFlow:
) as mock_get:
mock_get.return_value = torrents
mock_client = AsyncMock()
mock_client.add_torrent = AsyncMock(return_value=True)
mock_client.add_torrent = AsyncMock(return_value=AddResult.ADDED)
await engine.refresh_rss(mock_client)
# Only ep02 should be downloaded (ep01 already exists)

160
backend/src/test/test_rss_engine_new.py
View File

@@ -7,6 +7,7 @@ import pytest
import pytest_asyncio
from module.database import Database
from module.downloader import AddResult
from module.models import Torrent
from module.notification.events import DownloadFailureEvent, RssFailureEvent
from module.rss.engine import RSSEngine
@@ -185,6 +186,48 @@ class TestMatchTorrent:
assert result2 is not None
async def test_match_torrent_filter_rejected_leaves_bangumi_id_unset(
self, rss_engine
):
"""A filter-rejected torrent must NOT be associated with the bangumi,
or OffsetScanner would compute offsets from excluded episodes."""
bangumi = make_bangumi(title_raw="Mushoku Tensei", filter="720")
await rss_engine.db.bangumi.add(bangumi)
torrent = make_torrent(name="[Sub] Mushoku Tensei - 01 [720p].mkv")
result = rss_engine.match_torrent(
torrent, await rss_engine.db.bangumi.search_all()
)
assert result is None
assert torrent.bangumi_id is None
async def test_match_torrent_filter_passed_sets_bangumi_id(self, rss_engine):
"""A torrent passing a non-empty filter gets bangumi_id set."""
bangumi = make_bangumi(title_raw="Mushoku Tensei", filter="720")
await rss_engine.db.bangumi.add(bangumi)
torrent = make_torrent(name="[Sub] Mushoku Tensei - 01 [1080p].mkv")
result = rss_engine.match_torrent(
torrent, await rss_engine.db.bangumi.search_all()
)
assert result is not None
assert torrent.bangumi_id == result.id
async def test_match_torrent_empty_filter_sets_bangumi_id(self, rss_engine):
"""A torrent matched with an empty filter also gets bangumi_id set."""
bangumi = make_bangumi(title_raw="Mushoku Tensei", filter="")
await rss_engine.db.bangumi.add(bangumi)
torrent = make_torrent(name="[Sub] Mushoku Tensei - 01 [1080p].mkv")
result = rss_engine.match_torrent(
torrent, await rss_engine.db.bangumi.search_all()
)
assert result is not None
assert torrent.bangumi_id == result.id
# ---------------------------------------------------------------------------
# refresh_rss
@@ -212,7 +255,7 @@ class TestRefreshRss:
# Create a mock client
client = AsyncMock()
client.add_torrent = AsyncMock(return_value=True)
client.add_torrent = AsyncMock(return_value=AddResult.ADDED)
await rss_engine.refresh_rss(client)
@@ -346,7 +389,7 @@ class TestRefreshRssEvents:
) as mock_get:
mock_get.return_value = [new_torrent]
client = AsyncMock()
client.add_torrent = AsyncMock(return_value=False)
client.add_torrent = AsyncMock(return_value=AddResult.FAILED)
events = await rss_engine.refresh_rss(client)
@@ -355,8 +398,9 @@ class TestRefreshRssEvents:
assert events[0].official_title == bangumi.official_title
assert events[0].torrent_name == new_torrent.name
# The failed torrent is NOT persisted, so a later tick can retry it.
all_torrents = await rss_engine.db.torrent.search_all()
assert all_torrents[0].downloaded is False
assert all_torrents == []
async def test_add_torrent_success_returns_no_download_failure_event(
self, rss_engine, mock_qb_client
@@ -376,12 +420,116 @@ class TestRefreshRssEvents:
) as mock_get:
mock_get.return_value = [new_torrent]
client = AsyncMock()
client.add_torrent = AsyncMock(return_value=True)
client.add_torrent = AsyncMock(return_value=AddResult.ADDED)
events = await rss_engine.refresh_rss(client)
assert events == []
async def test_add_torrent_duplicate_marks_downloaded_without_event(
self, rss_engine
):
"""A torrent the client already has (DUPLICATE) is treated as success:
no failure notification, row persisted as downloaded."""
rss_item = make_rss_item(enabled=True)
await rss_engine.db.rss.add(rss_item)
bangumi = make_bangumi(title_raw="Mushoku Tensei", filter="")
await rss_engine.db.bangumi.add(bangumi)
new_torrent = Torrent(
name="[Sub] Mushoku Tensei - 12 [1080p].mkv",
url="https://example.com/ep12.torrent",
)
with patch.object(
RSSEngine, "_get_torrents", new_callable=AsyncMock
) as mock_get:
mock_get.return_value = [new_torrent]
client = AsyncMock()
client.add_torrent = AsyncMock(return_value=AddResult.DUPLICATE)
events = await rss_engine.refresh_rss(client)
assert events == []
all_torrents = await rss_engine.db.torrent.search_all()
assert len(all_torrents) == 1
assert all_torrents[0].downloaded is True
# ---------------------------------------------------------------------------
# refresh_rss retry semantics for failed adds
# ---------------------------------------------------------------------------
class TestRefreshRssRetry:
async def test_refresh_rss_failed_add_retried_on_next_tick(self, rss_engine):
"""A torrent whose add FAILED is re-attempted on the next refresh
(it must not be persisted, or check_new would filter it forever)."""
rss_item = make_rss_item(enabled=True)
await rss_engine.db.rss.add(rss_item)
bangumi = make_bangumi(title_raw="Mushoku Tensei", filter="")
await rss_engine.db.bangumi.add(bangumi)
def feed_torrent():
return Torrent(
name="[Sub] Mushoku Tensei - 12 [1080p].mkv",
url="https://example.com/ep12.torrent",
)
with patch.object(
RSSEngine, "_get_torrents", new_callable=AsyncMock
) as mock_get:
client = AsyncMock()
# Tick 1: transient failure
mock_get.return_value = [feed_torrent()]
client.add_torrent = AsyncMock(return_value=AddResult.FAILED)
await rss_engine.refresh_rss(client)
client.add_torrent.assert_called_once()
assert await rss_engine.db.torrent.search_all() == []
# Tick 2: same feed item is still there, add now succeeds
mock_get.return_value = [feed_torrent()]
client.add_torrent = AsyncMock(return_value=AddResult.ADDED)
await rss_engine.refresh_rss(client)
client.add_torrent.assert_called_once()
all_torrents = await rss_engine.db.torrent.search_all()
assert len(all_torrents) == 1
assert all_torrents[0].downloaded is True
async def test_refresh_rss_filter_rejected_persisted_and_not_retried(
self, rss_engine
):
"""Filter-rejected torrents keep the old behavior: persisted once
(downloaded=False, no bangumi association) and never re-processed."""
rss_item = make_rss_item(enabled=True)
await rss_engine.db.rss.add(rss_item)
bangumi = make_bangumi(title_raw="Mushoku Tensei", filter="720")
await rss_engine.db.bangumi.add(bangumi)
def feed_torrent():
return Torrent(
name="[Sub] Mushoku Tensei - 01 [720p].mkv",
url="https://example.com/ep01-720.torrent",
)
with patch.object(
RSSEngine, "_get_torrents", new_callable=AsyncMock
) as mock_get:
client = AsyncMock()
client.add_torrent = AsyncMock(return_value=AddResult.ADDED)
mock_get.return_value = [feed_torrent()]
await rss_engine.refresh_rss(client)
mock_get.return_value = [feed_torrent()]
await rss_engine.refresh_rss(client)
client.add_torrent.assert_not_called()
all_torrents = await rss_engine.db.torrent.search_all()
assert len(all_torrents) == 1
assert all_torrents[0].downloaded is False
assert all_torrents[0].bangumi_id is None
# ---------------------------------------------------------------------------
# add_rss
@@ -714,7 +862,7 @@ class TestRefreshRssPreferenceDedup:
) as mock_get:
mock_get.return_value = [preferred, other]
client = AsyncMock()
client.add_torrent = AsyncMock(return_value=True)
client.add_torrent = AsyncMock(return_value=AddResult.ADDED)
await rss_engine.refresh_rss(client)
@@ -749,7 +897,7 @@ class TestRefreshRssPreferenceDedup:
) as mock_get:
mock_get.return_value = [torrent_a, torrent_b]
client = AsyncMock()
client.add_torrent = AsyncMock(return_value=True)
client.add_torrent = AsyncMock(return_value=AddResult.ADDED)
await rss_engine.refresh_rss(client)