diff --git a/app/application/torrent/download.py b/app/application/torrent/download.py index c912247c4..28bd72cce 100644 --- a/app/application/torrent/download.py +++ b/app/application/torrent/download.py @@ -1,5 +1,6 @@ import datetime import re +import time from functools import lru_cache from pathlib import Path from typing import Any, Callable, Dict, List, Optional, Protocol, Tuple, Union @@ -24,6 +25,11 @@ from app.schemas.types import MediaType, SystemConfigKey _SIZE_UNIT = 1024 * 1024 +def _is_transient_torrent_status(status: int) -> bool: + """403 和服务端错误可能来自站点前置代理，不能据此认定种子已失效。""" + return status == 403 or 500 <= status <= 599 + + class TorrentResponsePort(Protocol): """声明种子下载用例实际读取的 HTTP 响应字段。""" @@ -141,6 +147,30 @@ class TorrentHelper: """初始化种子失败地址缓存""" self._invalid_torrents = TTLCache(region="invalid_torrents", maxsize=128, ttl=3600 * 24) + def _request_torrent_get(self, http_port: TorrentHttpPort, retry: bool, + **request_options: Any) -> Optional[TorrentResponsePort]: + """普通种子 GET 瞬时失败时退避 1、2 秒；签名换票和确认 POST 不在此重试。""" + for attempt in range(3 if retry else 1): + response = http_port.request(method="GET", **request_options) + if not retry or attempt == 2 or not self._should_retry_torrent_response(response): + return response + logger.warning(f"种子下载遇到瞬时失败，将进行第 {attempt + 1} 次重试") + time.sleep(attempt + 1) + return None + + def _should_retry_torrent_response(self, response: Optional[TorrentResponsePort]) -> bool: + """保留重定向、磁力和首次下载确认页，仅重试网络失败及无效的种子响应。""" + if response is None: + return True + if response.status_code != 200: + return _is_transient_torrent_status(response.status_code) + content = response.content + if content.startswith(b"magnet:"): + return False + if self.get_fileinfo_from_torrent_content(content)[1]: + return False + return "下载种子文件".encode("utf-8") not in content + def download_torrent(self, url: str, cookie: Optional[str] = None, ua: Optional[str] = None, @@ -155,7 +185,7 @@ class TorrentHelper: :param ua: 请求 User-Agent :param referer: 请求来源地址 :param proxy: 是否使用系统代理 - :param cache_invalid: 是否缓存失败地址；短时凭证地址必须关闭 + :param cache_invalid: 是否缓存失败地址并启用普通 GET 重试；短时凭证及换票地址必须关闭 :return: 种子缓存相对路径【用于索引缓存】, 种子内容、种子主目录、种子文件清单、错误信息 """ if url.startswith("magnet:"): @@ -181,19 +211,21 @@ class TorrentHelper: # 下载种子文件 config = get_chain_runtime_config_snapshot() http_port = _require_torrent_port() - req = http_port.request( - method="GET", url=url, cookie=cookie, ua=ua, referer=referer, + retry = cache_invalid and not torrent_rules.is_expiring_download_url(url) + req = self._request_torrent_get( + http_port, retry, url=url, cookie=cookie, ua=ua, referer=referer, proxies=config.proxy if proxy else None, allow_redirects=False, ) - while req and req.status_code in [301, 302]: + while req is not None and req.status_code in [301, 302]: url = req.headers['Location'] if url and url.startswith("magnet:"): return None, url, "", [], "获取到磁力链接" - req = http_port.request( - method="GET", url=url, cookie=cookie, ua=ua, referer=referer, + req = self._request_torrent_get( + http_port, retry and not torrent_rules.is_expiring_download_url(url), + url=url, cookie=cookie, ua=ua, referer=referer, proxies=config.proxy if proxy else None, allow_redirects=False, ) - if req and req.status_code == 200: + if req is not None and req.status_code == 200: if not req.content: return cache_path, None, "", [], "未下载到种子数据" # 解析内容格式 @@ -261,7 +293,7 @@ class TorrentHelper: return cache_path, None, "", [], "触发站点流控，请稍后重试" else: # 把错误的种子记下来，避免重复使用 - if cache_invalid: + if cache_invalid and not _is_transient_torrent_status(req.status_code): self.add_invalid(url) return cache_path, None, "", [], f"下载种子出错，状态码：{req.status_code}" diff --git a/app/chain/download/failure.py b/app/chain/download/failure.py index b89083d7e..31726ea05 100644 --- a/app/chain/download/failure.py +++ b/app/chain/download/failure.py @@ -26,7 +26,6 @@ DOWNLOAD_FAILURE_RESOURCE_TTL_SECONDS = 24 * 60 * 60 DOWNLOAD_FAILURE_TRANSIENT_TTL_SECONDS = 60 * 60 DOWNLOAD_FAILURE_RESOURCE_ERROR_KEYWORDS = ( "无法读取种子文件", - "下载种子内容为空", "无法获取下载地址", "种子下载失败", "torrent not found", @@ -130,7 +129,7 @@ class DownloadFailureOwner(_DownloadOwnerBase): @staticmethod def _download_failure_ttl(error_msg: Optional[str]) -> int: """ - 按失败原因确定资源冷却时间。 + 按失败原因确定资源冷却时间；泛化的空内容不能证明资源失效，使用瞬时冷却。 """ error_text = str(error_msg or "").lower() if any(keyword in error_text for keyword in DOWNLOAD_FAILURE_RESOURCE_ERROR_KEYWORDS): diff --git a/docs/testing.md b/docs/testing.md index 234103a24..140d7e36e 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -99,6 +99,14 @@ with patch.object(SomeModule, "fetch", new=AsyncMock(return_value=FAKE)): 这些场景只承诺已验证的 SMB 查询语义；为其他存储接入严格快照时，应在其 SDK/API 边界复现同样的状态转换，并评估目录时间语义、分页和请求成本。 +## 种子下载重试回归 + +种子下载重试的离线边界见 `tests/test_torrent_download_retry.py`：普通 GET 在网络失败、 +403/5xx、空响应或无法解析的响应后最多重试两次，退避为 1、2 秒。签名链接刷新、换票地址、 +首次下载确认 POST 和 429 流控保持各自的既有流程。测试替换 HTTP 端口与等待，不访问真实站点。 +瞬时状态不会写入 24 小时无效链接缓存；订阅记录的泛化“下载种子内容为空”使用 1 小时冷却， +明确资源失效仍保留 24 小时冷却。此策略不追溯修改已经写入的失败记录。 + ## 自隔离：用了什么，就还原什么 用例若修改了**进程级状态**——`sys.modules` 桩、单例（`Singleton._instances`）、`lru_cache`、环境变量、`settings` 字段——必须在用例或模块结束时还原。pytest 一次性导入全部测试模块，未还原的污染会扩散到后续用例，产生“单独跑过、一起跑挂”的测不准现象。 diff --git a/tests/test_torrent_download_retry.py b/tests/test_torrent_download_retry.py new file mode 100644 index 000000000..2c0304526 --- /dev/null +++ b/tests/test_torrent_download_retry.py @@ -0,0 +1,176 @@ +"""种子 GET 的有界重试、缓存与冷却策略离线回归。""" + +from unittest.mock import Mock + +import pytest +from requests import Response + +import app.application.torrent.download as download +from app.chain.download.failure import DownloadFailureOwner + +URL = "https://tracker.example/download.php?id=39" +CONTENT = b"d4:infod4:name10:Series.mkv6:lengthi1eee" + + +def _response(status=200, content=CONTENT): + """使用真实 Response 保证错误状态的布尔值不会掩盖状态码。""" + response = Response() + response.status_code = status + response._content = content + response.encoding = "utf-8" + return response + + +@pytest.fixture +def boundary(monkeypatch): + """隔离 HTTP、缓存和等待，保留真实种子解析与用例编排。""" + port = Mock() + monkeypatch.setattr(download, "_require_torrent_port", Mock(return_value=port)) + cache = Mock() + cache.get.return_value = None + monkeypatch.setattr(download, "FileCache", Mock(return_value=cache)) + sleep = Mock() + monkeypatch.setattr(download.time, "sleep", sleep) + helper = download.TorrentHelper() + helper.add_invalid = Mock() + return helper, port, cache, sleep + + +@pytest.mark.parametrize("failure", [None, 403, 500, 502, 503, 525, b"", b"d4:info", b"CF"]) +def test_transient_failure_recovers_without_invalid_cache(boundary, failure): + """网络异常的 None、代理错误和截断响应均可恢复，保持访问参数。""" + helper, port, cache, sleep = boundary + failed = _response(failure) if isinstance(failure, int) else ( + _response(content=failure) if failure is not None else None + ) + port.request.side_effect = [failed, _response()] + + result = helper.download_torrent(URL, cookie="session=test", ua="TestAgent", referer="https://tracker.example/") + + assert result[1:5] == (CONTENT, "", ["Series.mkv"], "") + assert port.request.call_count == 2 + assert port.request.call_args_list[0] == port.request.call_args_list[1] + sleep.assert_called_once_with(1) + helper.add_invalid.assert_not_called() + cache.set.assert_called_once() + + +@pytest.mark.parametrize("status", [None, 403, 525]) +def test_exhausted_retry_is_bounded_and_does_not_poison_url(boundary, status): + """失败耗尽后保留最终原因，总共三次请求且不写 24 小时缓存。""" + helper, port, cache, sleep = boundary + port.request.return_value = None if status is None else _response(status) + + result = helper.download_torrent(URL) + + assert result[1] is None + assert result[4] == ("无法打开链接" if status is None else f"下载种子出错，状态码：{status}") + assert port.request.call_count == 3 + assert [call.args[0] for call in sleep.call_args_list] == [1, 2] + helper.add_invalid.assert_not_called() + cache.set.assert_not_called() + + +@pytest.mark.parametrize("status", [401, 404, 410, 429]) +def test_permanent_errors_and_rate_limit_are_not_retried(boundary, status): + """确定失效地址保留无效缓存，429 保留站点流控语义且不短退避。""" + helper, port, _, sleep = boundary + port.request.return_value = _response(status) + + assert helper.download_torrent(URL)[1] is None + + port.request.assert_called_once() + sleep.assert_not_called() + assert helper.add_invalid.call_count == (0 if status == 429 else 1) + + +@pytest.mark.parametrize("signed,cache_invalid", [(True, True), (False, False)]) +def test_signed_and_indirect_links_keep_single_attempt(boundary, signed, cache_invalid): + """签名刷新和换票继续由上层持有重试预算。""" + helper, port, _, sleep = boundary + port.request.return_value = _response(403) + url = URL + "&t=1&sign=old" if signed else URL + + assert helper.download_torrent(url, cache_invalid=cache_invalid)[1] is None + + port.request.assert_called_once() + sleep.assert_not_called() + helper.add_invalid.assert_not_called() + + +def test_first_download_confirmation_post_is_not_retried(boundary): + """识别首次下载确认页后只提交一次 POST，不重复有副作用的操作。""" + helper, port, _, sleep = boundary + page = '下载种子文件
'.encode() + port.request.side_effect = [_response(content=page), _response(503)] + + assert helper.download_torrent(URL)[1] is None + + assert [call.kwargs["method"] for call in port.request.call_args_list] == ["GET", "POST"] + sleep.assert_not_called() + + +def test_redirected_get_retries_at_destination(boundary): + """重定向目标上的瞬时错误也重试，并保留原始 URL 的成功缓存键。""" + helper, port, cache, sleep = boundary + redirect = _response(302) + redirect.headers["Location"] = "https://tracker.example/file.torrent" + port.request.side_effect = [redirect, _response(525), _response()] + + assert helper.download_torrent(URL)[1] == CONTENT + + assert [call.kwargs["url"] for call in port.request.call_args_list] == [URL, redirect.headers["Location"], redirect.headers["Location"]] + sleep.assert_called_once_with(1) + cache.set.assert_called_once() + + +def test_second_retry_can_recover(boundary): + """第二次退避后成功时只缓存最终合法种子。""" + helper, port, cache, sleep = boundary + port.request.side_effect = [None, _response(content=b"d4:info"), _response()] + assert helper.download_torrent(URL)[1] == CONTENT + assert port.request.call_count == 3 + assert [call.args[0] for call in sleep.call_args_list] == [1, 2] + cache.set.assert_called_once() + + +@pytest.mark.parametrize("content", [b"", b"d4:info", b"CF"]) +def test_invalid_content_exhaustion_never_caches_response(boundary, content): + """空或无法解析的响应耗尽预算后仍不能进入种子及无效链接缓存。""" + helper, port, cache, _ = boundary + port.request.return_value = _response(content=content) + assert helper.download_torrent(URL)[1] is None + assert port.request.call_count == 3 + cache.set.assert_not_called() + helper.add_invalid.assert_not_called() + + +@pytest.mark.parametrize("content", [CONTENT, b"magnet:?xt=urn:btih:abc"]) +def test_valid_response_has_no_retry(boundary, content): + """合法种子和磁力响应只发起一次请求。""" + helper, port, _, sleep = boundary + port.request.return_value = _response(content=content) + assert helper.download_torrent(URL)[1] + port.request.assert_called_once() + sleep.assert_not_called() + + +def test_valid_cache_bypasses_network(boundary): + """已有合法种子缓存时不发起请求或退避。""" + helper, port, cache, sleep = boundary + cache.get.return_value = CONTENT + assert helper.download_torrent(URL)[1] == CONTENT + port.request.assert_not_called() + sleep.assert_not_called() + + +@pytest.mark.parametrize("error,ttl", [ + ("下载种子内容为空", 3600), + ("无法打开链接", 3600), + ("下载种子出错，状态码：525", 3600), + ("无法读取种子文件", 86400), + ("torrent not found", 86400), +]) +def test_generic_empty_download_is_transient(error, ttl): + """空内容不证明资源失效；明确文件损坏或删除继续按一天冷却。""" + assert DownloadFailureOwner._download_failure_ttl(error) == ttl